diff --git a/infra_lib/telemetry/.vpython3 b/infra_lib/telemetry/.vpython3 new file mode 100644 index 000000000..02ef99a4c --- /dev/null +++ b/infra_lib/telemetry/.vpython3 @@ -0,0 +1,96 @@ +# This includes the wheels necessary to run the telemetry and to run pytest +# +# Read more about `vpython` and how to modify this file here: +# https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md + +python_version: "3.8" + +wheel: < + name: "infra/python/wheels/typing-extensions-py3" + version: "version:4.0.1" +> + +wheel: < + name: "infra/python/wheels/zipp-py3" + version: "version:3.17.0" +> + +wheel: < + name: "infra/python/wheels/importlib-metadata-py3" + version: "version:6.0.0" +> + +wheel: < + name: "infra/python/wheels/wrapt-py3" + version: "version:1.15.0" +> + +wheel: < + name: "infra/python/wheels/deprecated-py3" + version: "version:1.2.13" +> + +wheel: < + name: "infra/python/wheels/opentelemetry-semantic-conventions-py3" + version: "version:0.39b0" +> + +wheel: < + name: "infra/python/wheels/opentelemetry-api-py3" + version: "version:1.18.0" +> + +wheel: < + name: "infra/python/wheels/opentelemetry-sdk-py3" + version: "version:1.18.0" +> + +wheel: < + name: "infra/python/wheels/protobuf-py3" + version: "version:4.25.1" +> + +wheel: < + name: "infra/python/wheels/googleapis-common-protos-py2_py3" + version: "version:1.61.0" +> + +wheel: < + name: "infra/python/wheels/packaging-py3" + version: "version:21.3" +> + +wheel: < + name: "infra/python/wheels/pyparsing-py2_py3" + version: "version:2.4.7" +> + +wheel: < + name: "infra/python/wheels/iniconfig-py3" + version: "version:1.1.1" +> + +wheel: < + name: "infra/python/wheels/tomli-py3" + version: "version:2.0.1" +> + +wheel: < + name: "infra/python/wheels/exceptiongroup-py3" + version: "version:1.1.2" +> + +wheel: < + name: "infra/python/wheels/pluggy-py3" + version: "version:0.13.1" +> + +wheel: < + name: "infra/python/wheels/pytest-py3" + version: "version:7.3.1" +> + +wheel: < + name: "infra/python/wheels/pytest-asyncio-py3" + version: "version:0.19.0" +> diff --git a/infra_lib/telemetry/PRESUBMIT.py b/infra_lib/telemetry/PRESUBMIT.py new file mode 100644 index 000000000..ab2068470 --- /dev/null +++ b/infra_lib/telemetry/PRESUBMIT.py @@ -0,0 +1,16 @@ +# Copyright 2024 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +PRESUBMIT_VERSION = '2.0.0' + + +def CheckTests(input_api, output_api): + if input_api.platform.startswith(('cygwin', 'win32')): + return [] + return input_api.RunTests([ + input_api.Command(name='telemetry', + cmd=['vpython3', '-m', 'pytest', '.'], + kwargs={'cwd': input_api.PresubmitLocalPath()}, + message=output_api.PresubmitError) + ]) diff --git a/infra_lib/telemetry/__init__.py b/infra_lib/telemetry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infra_lib/telemetry/detector.py b/infra_lib/telemetry/detector.py new file mode 100644 index 000000000..bf7bc02dd --- /dev/null +++ b/infra_lib/telemetry/detector.py @@ -0,0 +1,158 @@ +# Copyright 2024 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Defines the ResourceDetector to capture resource properties.""" + +import logging +import os +from pathlib import Path +import platform +import sys +from typing import Optional, Sequence + +from opentelemetry.sdk import resources + +CPU_ARCHITECTURE = "cpu.architecture" +CPU_NAME = "cpu.name" +CPU_COUNT = "cpu.count" +HOST_TYPE = "host.type" +MEMORY_SWAP_TOTAL = "memory.swap.total" +MEMORY_TOTAL = "memory.total" +PROCESS_CWD = "process.cwd" +PROCESS_RUNTIME_API_VERSION = "process.runtime.apiversion" +PROCESS_ENV = "process.env" +OS_NAME = "os.name" +DMI_PATH = Path("/sys/class/dmi/id/product_name") +GCE_DMI = "Google Compute Engine" +PROC_MEMINFO_PATH = Path("/proc/meminfo") + + +class ProcessDetector(resources.ResourceDetector): + """ResourceDetector to capture information about the process.""" + + def __init__(self, allowed_env: Optional[Sequence[str]] = None) -> None: + super().__init__() + self._allowed_env = allowed_env or ["USE"] + + def detect(self) -> resources.Resource: + env = os.environ + resource = { + PROCESS_CWD: os.getcwd(), + PROCESS_RUNTIME_API_VERSION: sys.api_version, + resources.PROCESS_PID: os.getpid(), + resources.PROCESS_OWNER: os.geteuid(), + resources.PROCESS_EXECUTABLE_NAME: Path(sys.executable).name, + resources.PROCESS_EXECUTABLE_PATH: sys.executable, + resources.PROCESS_COMMAND: sys.argv[0], + resources.PROCESS_COMMAND_ARGS: sys.argv[1:], + } + resource.update({ + f"{PROCESS_ENV}.{k}": env[k] + for k in self._allowed_env if k in env + }) + + return resources.Resource(resource) + + +class SystemDetector(resources.ResourceDetector): + """ResourceDetector to capture information about system.""" + + def detect(self) -> resources.Resource: + host_type = "UNKNOWN" + + if DMI_PATH.exists(): + host_type = DMI_PATH.read_text(encoding="utf-8") + + mem_info = MemoryInfo() + + resource = { + CPU_ARCHITECTURE: platform.machine(), + CPU_COUNT: os.cpu_count(), + CPU_NAME: platform.processor(), + HOST_TYPE: host_type.strip(), + MEMORY_SWAP_TOTAL: mem_info.total_swap_memory, + MEMORY_TOTAL: mem_info.total_physical_ram, + OS_NAME: os.name, + resources.OS_TYPE: platform.system(), + resources.OS_DESCRIPTION: platform.platform(), + } + + return resources.Resource(resource) + + +class MemoryInfo: + """Read machine memory info from /proc/meminfo.""" + + # Prefixes for the /proc/meminfo file that we care about. + MEMINFO_VIRTUAL_MEMORY_TOTAL = "VmallocTotal" + MEMINFO_PHYSICAL_RAM_TOTAL = "MemTotal" + MEMINFO_SWAP_MEMORY_TOTAL = "SwapTotal" + + def __init__(self) -> None: + self._total_physical_ram = 0 + self._total_virtual_memory = 0 + self._total_swap_memory = 0 + try: + contents = PROC_MEMINFO_PATH.read_text(encoding="utf-8") + except OSError as e: + logging.warning("Encountered an issue reading /proc/meminfo: %s", e) + return + + for line in contents.splitlines(): + if line.startswith(self.MEMINFO_SWAP_MEMORY_TOTAL): + self._total_swap_memory = self._get_mem_value(line) + elif line.startswith(self.MEMINFO_VIRTUAL_MEMORY_TOTAL): + self._total_virtual_memory = self._get_mem_value(line) + elif line.startswith(self.MEMINFO_PHYSICAL_RAM_TOTAL): + self._total_physical_ram = self._get_mem_value(line) + + @property + def total_physical_ram(self) -> int: + return self._total_physical_ram + + @property + def total_virtual_memory(self) -> int: + return self._total_virtual_memory + + @property + def total_swap_memory(self) -> int: + return self._total_swap_memory + + def _get_mem_value(self, line: str) -> int: + """Reads an individual line from /proc/meminfo and returns the size. + + This function also converts the read value from kibibytes to bytes + when the read value has a unit provided for memory size. + + The specification information for /proc files, including meminfo, can + be found at + https://www.kernel.org/doc/Documentation/filesystems/proc.txt. + + Args: + line: The text line read from /proc/meminfo. + + Returns: + The integer value after conversion. + """ + components = line.split() + if len(components) == 1: + logging.warning( + "Unexpected /proc/meminfo entry with no label:number value was " + "provided. Value read: '%s'", + line, + ) + return 0 + size = int(components[1]) + if len(components) == 2: + return size + # The RHEL and kernel.org specs for /proc/meminfo doesn't give any + # indication that a memory unit besides kB (kibibytes) is expected, + # except in the cases of page counts, where no unit is provided. + if components[2] != "kB": + logging.warning( + "Unit for memory consumption in /proc/meminfo does " + "not conform to expectations. Please review the " + "read value: %s", + line, + ) + return size * 1024 diff --git a/infra_lib/telemetry/detector_unittest.py b/infra_lib/telemetry/detector_unittest.py new file mode 100644 index 000000000..868f13574 --- /dev/null +++ b/infra_lib/telemetry/detector_unittest.py @@ -0,0 +1,236 @@ +# Copyright 2024 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""The tests for resource detector classes.""" + +import getpass +import logging +import os +from pathlib import Path +import platform +import sys + +from . import detector +from opentelemetry.sdk import resources + + +def mock_exists(path: os.PathLike, val: bool): + """Mock Path.exists for specified path.""" + + exists = Path.exists + + def _mock_exists(*args, **kwargs): + if args[0] == path: + return val + return exists(*args, **kwargs) + + return _mock_exists + + +def mock_read_text(path: os.PathLike, val: str): + """Mock Path.read_text for specified path.""" + + real_read_text = Path.read_text + + def _mock_read_text(*args, **kwargs): + if args[0] == path: + return val + return real_read_text(*args, **kwargs) + + return _mock_read_text + + +def test_process_info_capture() -> None: + """Test that ProcessDetector captures correct process info.""" + env_var = list(os.environ.keys())[0] + + d = detector.ProcessDetector(allowed_env=[env_var]) + attrs = d.detect().attributes + + assert attrs[resources.PROCESS_PID] == os.getpid() + assert attrs[detector.PROCESS_CWD] == os.getcwd() + assert attrs[resources.PROCESS_COMMAND] == sys.argv[0] + assert attrs[resources.PROCESS_COMMAND_ARGS] == tuple(sys.argv[1:]) + assert attrs[resources.PROCESS_EXECUTABLE_NAME] == Path(sys.executable).name + assert attrs[resources.PROCESS_EXECUTABLE_PATH] == sys.executable + assert attrs[f"process.env.{env_var}"] == os.environ[env_var] + + +def test_system_info_captured(monkeypatch) -> None: + """Test that SystemDetector captures the correct system info.""" + + monkeypatch.setattr(getpass, "getuser", lambda: "someuser") + monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, True)) + monkeypatch.setattr( + Path, + "read_text", + mock_read_text(detector.DMI_PATH, detector.GCE_DMI), + ) + + d = detector.SystemDetector() + attrs = d.detect().attributes + + assert attrs[detector.CPU_COUNT] == os.cpu_count() + assert attrs[detector.HOST_TYPE] == "Google Compute Engine" + assert attrs[detector.OS_NAME] == os.name + assert attrs[resources.OS_TYPE] == platform.system() + assert attrs[resources.OS_DESCRIPTION] == platform.platform() + assert attrs[detector.CPU_ARCHITECTURE] == platform.machine() + assert attrs[detector.CPU_NAME] == platform.processor() + + +def test_memory_info_class(monkeypatch) -> None: + proc_meminfo_contents = """ +SwapTotal: 15 kB +VmallocTotal: 25 kB +MemTotal: 35 kB + """ + monkeypatch.setattr(Path, "exists", + mock_exists(detector.PROC_MEMINFO_PATH, True)) + monkeypatch.setattr( + Path, + "read_text", + mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents), + ) + + m = detector.MemoryInfo() + assert m.total_swap_memory == 15 * 1024 + assert m.total_physical_ram == 35 * 1024 + assert m.total_virtual_memory == 25 * 1024 + + +def test_memory_info_class_warns_on_unexpected_unit(monkeypatch, + caplog) -> None: + proc_meminfo_contents = """ +SwapTotal: 15 mB +VmallocTotal: 25 gB +MemTotal: 35 tB + """ + monkeypatch.setattr(Path, "exists", + mock_exists(detector.PROC_MEMINFO_PATH, True)) + monkeypatch.setattr( + Path, + "read_text", + mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents), + ) + caplog.set_level(logging.WARNING) + + m = detector.MemoryInfo() + assert "Unit for memory consumption in /proc/meminfo" in caplog.text + # We do not attempt to correct unexpected units + assert m.total_swap_memory == 15 * 1024 + assert m.total_physical_ram == 35 * 1024 + assert m.total_virtual_memory == 25 * 1024 + + +def test_memory_info_class_no_units(monkeypatch) -> None: + proc_meminfo_contents = """ +SwapTotal: 15 + """ + monkeypatch.setattr(Path, "exists", + mock_exists(detector.PROC_MEMINFO_PATH, True)) + monkeypatch.setattr( + Path, + "read_text", + mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents), + ) + + m = detector.MemoryInfo() + assert m.total_swap_memory == 15 + + +def test_memory_info_class_no_provided_value(monkeypatch, caplog) -> None: + proc_meminfo_contents = """ +SwapTotal: + """ + monkeypatch.setattr(Path, "exists", + mock_exists(detector.PROC_MEMINFO_PATH, True)) + monkeypatch.setattr( + Path, + "read_text", + mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents), + ) + caplog.set_level(logging.WARNING) + + detector.MemoryInfo() + assert "Unexpected /proc/meminfo entry with no label:number" in caplog.text + + +def test_system_info_to_capture_memory_resources(monkeypatch) -> None: + proc_meminfo_contents = """ +SwapTotal: 15 kB +VmallocTotal: 25 kB +MemTotal: 35 kB + """ + monkeypatch.setattr(Path, "exists", + mock_exists(detector.PROC_MEMINFO_PATH, True)) + monkeypatch.setattr( + Path, + "read_text", + mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents), + ) + + d = detector.SystemDetector() + attrs = d.detect().attributes + + assert attrs[detector.MEMORY_TOTAL] == 35 * 1024 + assert attrs[detector.MEMORY_SWAP_TOTAL] == 15 * 1024 + + +def test_system_info_to_capture_host_type_bot(monkeypatch) -> None: + """Test that SystemDetector captures host type as Google Compute Engine.""" + + monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, True)) + monkeypatch.setattr( + Path, + "read_text", + mock_read_text(detector.DMI_PATH, detector.GCE_DMI), + ) + + d = detector.SystemDetector() + attrs = d.detect().attributes + + assert attrs[detector.CPU_COUNT] == os.cpu_count() + assert attrs[detector.HOST_TYPE] == detector.GCE_DMI + assert attrs[detector.OS_NAME] == os.name + assert attrs[resources.OS_TYPE] == platform.system() + assert attrs[resources.OS_DESCRIPTION] == platform.platform() + assert attrs[detector.CPU_ARCHITECTURE] == platform.machine() + assert attrs[detector.CPU_NAME] == platform.processor() + + +def test_system_info_to_capture_host_type_from_dmi(monkeypatch) -> None: + """Test that SystemDetector captures dmi product name as host type.""" + + monkeypatch.setattr(getpass, "getuser", lambda: "someuser") + monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, True)) + monkeypatch.setattr(Path, "read_text", + mock_read_text(detector.DMI_PATH, "SomeId")) + + d = detector.SystemDetector() + attrs = d.detect().attributes + + assert attrs[detector.CPU_COUNT] == os.cpu_count() + assert attrs[detector.HOST_TYPE] == "SomeId" + assert attrs[detector.OS_NAME] == os.name + assert attrs[resources.OS_TYPE] == platform.system() + assert attrs[resources.OS_DESCRIPTION] == platform.platform() + assert attrs[detector.CPU_ARCHITECTURE] == platform.machine() + assert attrs[detector.CPU_NAME] == platform.processor() + + +def test_system_info_to_capture_host_type_unknown(monkeypatch) -> None: + """Test that SystemDetector captures host type as UNKNOWN.""" + + monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, False)) + + d = detector.SystemDetector() + attrs = d.detect().attributes + + assert attrs[detector.CPU_COUNT] == os.cpu_count() + assert attrs[detector.HOST_TYPE] == "UNKNOWN" + assert attrs[detector.OS_NAME] == os.name + assert attrs[resources.OS_TYPE] == platform.system() + assert attrs[resources.OS_DESCRIPTION] == platform.platform() + assert attrs[detector.CPU_ARCHITECTURE] == platform.machine() + assert attrs[detector.CPU_NAME] == platform.processor() diff --git a/infra_lib/telemetry/pytest.ini b/infra_lib/telemetry/pytest.ini new file mode 100644 index 000000000..a7760ec09 --- /dev/null +++ b/infra_lib/telemetry/pytest.ini @@ -0,0 +1,8 @@ +# Copyright 2024 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Syntax: https://docs.pytest.org/en/latest/customize.html + +[pytest] +python_files = *_unittest.py