aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add the detector to telemetry library

Browse Source 
The detector can be a pretty direct port from chromite. A lot of these
metrics aren't likely neccesary but will be good to have complete
trace protos rather than missings parts. This also creates the
presubmit and .vpython3 file necessary to run the unit tests

Bug: 326277821
Change-Id: I4dbeabbbced4715527201eca888948b07b6004ca
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5840599
Commit-Queue: Struan Shrimpton <sshrimp@google.com>
Reviewed-by: Terrence Reilly <treilly@google.com>
changes/99/5840599/4
Struan Shrimpton 6 months ago committed by LUCI CQ
parent c246cf3e84
commit c36eb432d9
6 changed files with 514 additions and 0 deletions
Show Stats Download Patch File Download Diff File

96
infra_lib/telemetry/.vpython3
Unescape Escape View File

@ -0,0 +1,96 @@
# This includes the wheels necessary to run the telemetry and to run pytest
#
# Read more about `vpython` and how to modify this file here:
# https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
python_version: "3.8"
wheel: <
name: "infra/python/wheels/typing-extensions-py3"
version: "version:4.0.1"
>
wheel: <
name: "infra/python/wheels/zipp-py3"
version: "version:3.17.0"
>
wheel: <
name: "infra/python/wheels/importlib-metadata-py3"
version: "version:6.0.0"
>
wheel: <
name: "infra/python/wheels/wrapt-py3"
version: "version:1.15.0"
>
wheel: <
name: "infra/python/wheels/deprecated-py3"
version: "version:1.2.13"
>
wheel: <
name: "infra/python/wheels/opentelemetry-semantic-conventions-py3"
version: "version:0.39b0"
>
wheel: <
name: "infra/python/wheels/opentelemetry-api-py3"
version: "version:1.18.0"
>
wheel: <
name: "infra/python/wheels/opentelemetry-sdk-py3"
version: "version:1.18.0"
>
wheel: <
name: "infra/python/wheels/protobuf-py3"
version: "version:4.25.1"
>
wheel: <
name: "infra/python/wheels/googleapis-common-protos-py2_py3"
version: "version:1.61.0"
>
wheel: <
name: "infra/python/wheels/packaging-py3"
version: "version:21.3"
>
wheel: <
name: "infra/python/wheels/pyparsing-py2_py3"
version: "version:2.4.7"
>
wheel: <
name: "infra/python/wheels/iniconfig-py3"
version: "version:1.1.1"
>
wheel: <
name: "infra/python/wheels/tomli-py3"
version: "version:2.0.1"
>
wheel: <
name: "infra/python/wheels/exceptiongroup-py3"
version: "version:1.1.2"
>
wheel: <
name: "infra/python/wheels/pluggy-py3"
version: "version:0.13.1"
>
wheel: <
name: "infra/python/wheels/pytest-py3"
version: "version:7.3.1"
>
wheel: <
name: "infra/python/wheels/pytest-asyncio-py3"
version: "version:0.19.0"
>

16
infra_lib/telemetry/PRESUBMIT.py
Unescape Escape View File

@ -0,0 +1,16 @@
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
PRESUBMIT_VERSION = '2.0.0'
def CheckTests(input_api, output_api):
if input_api.platform.startswith(('cygwin', 'win32')):
return []
return input_api.RunTests([
input_api.Command(name='telemetry',
cmd=['vpython3', '-m', 'pytest', '.'],
kwargs={'cwd': input_api.PresubmitLocalPath()},
message=output_api.PresubmitError)
])

0
infra_lib/telemetry/__init__.py
Unescape Escape View File

158
infra_lib/telemetry/detector.py
Unescape Escape View File

@ -0,0 +1,158 @@
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Defines the ResourceDetector to capture resource properties."""
import logging
import os
from pathlib import Path
import platform
import sys
from typing import Optional, Sequence
from opentelemetry.sdk import resources
CPU_ARCHITECTURE = "cpu.architecture"
CPU_NAME = "cpu.name"
CPU_COUNT = "cpu.count"
HOST_TYPE = "host.type"
MEMORY_SWAP_TOTAL = "memory.swap.total"
MEMORY_TOTAL = "memory.total"
PROCESS_CWD = "process.cwd"
PROCESS_RUNTIME_API_VERSION = "process.runtime.apiversion"
PROCESS_ENV = "process.env"
OS_NAME = "os.name"
DMI_PATH = Path("/sys/class/dmi/id/product_name")
GCE_DMI = "Google Compute Engine"
PROC_MEMINFO_PATH = Path("/proc/meminfo")
class ProcessDetector(resources.ResourceDetector):
"""ResourceDetector to capture information about the process."""
def __init__(self, allowed_env: Optional[Sequence[str]] = None) -> None:
super().__init__()
self._allowed_env = allowed_env or ["USE"]
def detect(self) -> resources.Resource:
env = os.environ
resource = {
PROCESS_CWD: os.getcwd(),
PROCESS_RUNTIME_API_VERSION: sys.api_version,
resources.PROCESS_PID: os.getpid(),
resources.PROCESS_OWNER: os.geteuid(),
resources.PROCESS_EXECUTABLE_NAME: Path(sys.executable).name,
resources.PROCESS_EXECUTABLE_PATH: sys.executable,
resources.PROCESS_COMMAND: sys.argv[0],
resources.PROCESS_COMMAND_ARGS: sys.argv[1:],
}
resource.update({
f"{PROCESS_ENV}.{k}": env[k]
for k in self._allowed_env if k in env
})
return resources.Resource(resource)
class SystemDetector(resources.ResourceDetector):
"""ResourceDetector to capture information about system."""
def detect(self) -> resources.Resource:
host_type = "UNKNOWN"
if DMI_PATH.exists():
host_type = DMI_PATH.read_text(encoding="utf-8")
mem_info = MemoryInfo()
resource = {
CPU_ARCHITECTURE: platform.machine(),
CPU_COUNT: os.cpu_count(),
CPU_NAME: platform.processor(),
HOST_TYPE: host_type.strip(),
MEMORY_SWAP_TOTAL: mem_info.total_swap_memory,
MEMORY_TOTAL: mem_info.total_physical_ram,
OS_NAME: os.name,
resources.OS_TYPE: platform.system(),
resources.OS_DESCRIPTION: platform.platform(),
}
return resources.Resource(resource)
class MemoryInfo:
"""Read machine memory info from /proc/meminfo."""
# Prefixes for the /proc/meminfo file that we care about.
MEMINFO_VIRTUAL_MEMORY_TOTAL = "VmallocTotal"
MEMINFO_PHYSICAL_RAM_TOTAL = "MemTotal"
MEMINFO_SWAP_MEMORY_TOTAL = "SwapTotal"
def __init__(self) -> None:
self._total_physical_ram = 0
self._total_virtual_memory = 0
self._total_swap_memory = 0
try:
contents = PROC_MEMINFO_PATH.read_text(encoding="utf-8")
except OSError as e:
logging.warning("Encountered an issue reading /proc/meminfo: %s", e)
return
for line in contents.splitlines():
if line.startswith(self.MEMINFO_SWAP_MEMORY_TOTAL):
self._total_swap_memory = self._get_mem_value(line)
elif line.startswith(self.MEMINFO_VIRTUAL_MEMORY_TOTAL):
self._total_virtual_memory = self._get_mem_value(line)
elif line.startswith(self.MEMINFO_PHYSICAL_RAM_TOTAL):
self._total_physical_ram = self._get_mem_value(line)
@property
def total_physical_ram(self) -> int:
return self._total_physical_ram
@property
def total_virtual_memory(self) -> int:
return self._total_virtual_memory
@property
def total_swap_memory(self) -> int:
return self._total_swap_memory
def _get_mem_value(self, line: str) -> int:
"""Reads an individual line from /proc/meminfo and returns the size.
This function also converts the read value from kibibytes to bytes
when the read value has a unit provided for memory size.
The specification information for /proc files, including meminfo, can
be found at
https://www.kernel.org/doc/Documentation/filesystems/proc.txt.
Args:
line: The text line read from /proc/meminfo.
Returns:
The integer value after conversion.
"""
components = line.split()
if len(components) == 1:
logging.warning(
"Unexpected /proc/meminfo entry with no label:number value was "
"provided. Value read: '%s'",
line,
)
return 0
size = int(components[1])
if len(components) == 2:
return size
# The RHEL and kernel.org specs for /proc/meminfo doesn't give any
# indication that a memory unit besides kB (kibibytes) is expected,
# except in the cases of page counts, where no unit is provided.
if components[2] != "kB":
logging.warning(
"Unit for memory consumption in /proc/meminfo does "
"not conform to expectations. Please review the "
"read value: %s",
line,
)
return size * 1024

236
infra_lib/telemetry/detector_unittest.py
Unescape Escape View File

@ -0,0 +1,236 @@
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""The tests for resource detector classes."""
import getpass
import logging
import os
from pathlib import Path
import platform
import sys
from . import detector
from opentelemetry.sdk import resources
def mock_exists(path: os.PathLike, val: bool):
"""Mock Path.exists for specified path."""
exists = Path.exists
def _mock_exists(*args, **kwargs):
if args[0] == path:
return val
return exists(*args, **kwargs)
return _mock_exists
def mock_read_text(path: os.PathLike, val: str):
"""Mock Path.read_text for specified path."""
real_read_text = Path.read_text
def _mock_read_text(*args, **kwargs):
if args[0] == path:
return val
return real_read_text(*args, **kwargs)
return _mock_read_text
def test_process_info_capture() -> None:
"""Test that ProcessDetector captures correct process info."""
env_var = list(os.environ.keys())[0]
d = detector.ProcessDetector(allowed_env=[env_var])
attrs = d.detect().attributes
assert attrs[resources.PROCESS_PID] == os.getpid()
assert attrs[detector.PROCESS_CWD] == os.getcwd()
assert attrs[resources.PROCESS_COMMAND] == sys.argv[0]
assert attrs[resources.PROCESS_COMMAND_ARGS] == tuple(sys.argv[1:])
assert attrs[resources.PROCESS_EXECUTABLE_NAME] == Path(sys.executable).name
assert attrs[resources.PROCESS_EXECUTABLE_PATH] == sys.executable
assert attrs[f"process.env.{env_var}"] == os.environ[env_var]
def test_system_info_captured(monkeypatch) -> None:
"""Test that SystemDetector captures the correct system info."""
monkeypatch.setattr(getpass, "getuser", lambda: "someuser")
monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, True))
monkeypatch.setattr(
Path,
"read_text",
mock_read_text(detector.DMI_PATH, detector.GCE_DMI),
)
d = detector.SystemDetector()
attrs = d.detect().attributes
assert attrs[detector.CPU_COUNT] == os.cpu_count()
assert attrs[detector.HOST_TYPE] == "Google Compute Engine"
assert attrs[detector.OS_NAME] == os.name
assert attrs[resources.OS_TYPE] == platform.system()
assert attrs[resources.OS_DESCRIPTION] == platform.platform()
assert attrs[detector.CPU_ARCHITECTURE] == platform.machine()
assert attrs[detector.CPU_NAME] == platform.processor()
def test_memory_info_class(monkeypatch) -> None:
proc_meminfo_contents = """
SwapTotal: 15 kB
VmallocTotal: 25 kB
MemTotal: 35 kB
"""
monkeypatch.setattr(Path, "exists",
mock_exists(detector.PROC_MEMINFO_PATH, True))
monkeypatch.setattr(
Path,
"read_text",
mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents),
)
m = detector.MemoryInfo()
assert m.total_swap_memory == 15 * 1024
assert m.total_physical_ram == 35 * 1024
assert m.total_virtual_memory == 25 * 1024
def test_memory_info_class_warns_on_unexpected_unit(monkeypatch,
caplog) -> None:
proc_meminfo_contents = """
SwapTotal: 15 mB
VmallocTotal: 25 gB
MemTotal: 35 tB
"""
monkeypatch.setattr(Path, "exists",
mock_exists(detector.PROC_MEMINFO_PATH, True))
monkeypatch.setattr(
Path,
"read_text",
mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents),
)
caplog.set_level(logging.WARNING)
m = detector.MemoryInfo()
assert "Unit for memory consumption in /proc/meminfo" in caplog.text
# We do not attempt to correct unexpected units
assert m.total_swap_memory == 15 * 1024
assert m.total_physical_ram == 35 * 1024
assert m.total_virtual_memory == 25 * 1024
def test_memory_info_class_no_units(monkeypatch) -> None:
proc_meminfo_contents = """
SwapTotal: 15
"""
monkeypatch.setattr(Path, "exists",
mock_exists(detector.PROC_MEMINFO_PATH, True))
monkeypatch.setattr(
Path,
"read_text",
mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents),
)
m = detector.MemoryInfo()
assert m.total_swap_memory == 15
def test_memory_info_class_no_provided_value(monkeypatch, caplog) -> None:
proc_meminfo_contents = """
SwapTotal:
"""
monkeypatch.setattr(Path, "exists",
mock_exists(detector.PROC_MEMINFO_PATH, True))
monkeypatch.setattr(
Path,
"read_text",
mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents),
)
caplog.set_level(logging.WARNING)
detector.MemoryInfo()
assert "Unexpected /proc/meminfo entry with no label:number" in caplog.text
def test_system_info_to_capture_memory_resources(monkeypatch) -> None:
proc_meminfo_contents = """
SwapTotal: 15 kB
VmallocTotal: 25 kB
MemTotal: 35 kB
"""
monkeypatch.setattr(Path, "exists",
mock_exists(detector.PROC_MEMINFO_PATH, True))
monkeypatch.setattr(
Path,
"read_text",
mock_read_text(detector.PROC_MEMINFO_PATH, proc_meminfo_contents),
)
d = detector.SystemDetector()
attrs = d.detect().attributes
assert attrs[detector.MEMORY_TOTAL] == 35 * 1024
assert attrs[detector.MEMORY_SWAP_TOTAL] == 15 * 1024
def test_system_info_to_capture_host_type_bot(monkeypatch) -> None:
"""Test that SystemDetector captures host type as Google Compute Engine."""
monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, True))
monkeypatch.setattr(
Path,
"read_text",
mock_read_text(detector.DMI_PATH, detector.GCE_DMI),
)
d = detector.SystemDetector()
attrs = d.detect().attributes
assert attrs[detector.CPU_COUNT] == os.cpu_count()
assert attrs[detector.HOST_TYPE] == detector.GCE_DMI
assert attrs[detector.OS_NAME] == os.name
assert attrs[resources.OS_TYPE] == platform.system()
assert attrs[resources.OS_DESCRIPTION] == platform.platform()
assert attrs[detector.CPU_ARCHITECTURE] == platform.machine()
assert attrs[detector.CPU_NAME] == platform.processor()
def test_system_info_to_capture_host_type_from_dmi(monkeypatch) -> None:
"""Test that SystemDetector captures dmi product name as host type."""
monkeypatch.setattr(getpass, "getuser", lambda: "someuser")
monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, True))
monkeypatch.setattr(Path, "read_text",
mock_read_text(detector.DMI_PATH, "SomeId"))
d = detector.SystemDetector()
attrs = d.detect().attributes
assert attrs[detector.CPU_COUNT] == os.cpu_count()
assert attrs[detector.HOST_TYPE] == "SomeId"
assert attrs[detector.OS_NAME] == os.name
assert attrs[resources.OS_TYPE] == platform.system()
assert attrs[resources.OS_DESCRIPTION] == platform.platform()
assert attrs[detector.CPU_ARCHITECTURE] == platform.machine()
assert attrs[detector.CPU_NAME] == platform.processor()
def test_system_info_to_capture_host_type_unknown(monkeypatch) -> None:
"""Test that SystemDetector captures host type as UNKNOWN."""
monkeypatch.setattr(Path, "exists", mock_exists(detector.DMI_PATH, False))
d = detector.SystemDetector()
attrs = d.detect().attributes
assert attrs[detector.CPU_COUNT] == os.cpu_count()
assert attrs[detector.HOST_TYPE] == "UNKNOWN"
assert attrs[detector.OS_NAME] == os.name
assert attrs[resources.OS_TYPE] == platform.system()
assert attrs[resources.OS_DESCRIPTION] == platform.platform()
assert attrs[detector.CPU_ARCHITECTURE] == platform.machine()
assert attrs[detector.CPU_NAME] == platform.processor()

8
infra_lib/telemetry/pytest.ini
Unescape Escape View File

@ -0,0 +1,8 @@
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Syntax: https://docs.pytest.org/en/latest/customize.html
[pytest]
python_files = *_unittest.py
Write Preview
Loading…
Cancel
Save