{ "$schema": "https://firefox-ci-tc.services.mozilla.com/schemas/common/logs-reference-v0.json#", "serviceName": "worker-manager", "types": [ { "description": "A timer and audit for express API endpoints.\nYou can combine this with auth audit logs to get\na complete picture of what was authorized when.\n\nHere, anything that is not public should have authenticated\nand if it authenticated, we will tell the clientId here. Given\nthat, it is not necessarily true that the endpoint was\n_authorized_. You can tell that by the statusCode.", "fields": { "apiVersion": "The version of the API (e.g., `v1`)", "authenticated": "Will be `true` if the request has successfully authenticated (`auth-success` or `no-auth`) but\n`false` if req.authenticate() has not been called (for example when no scopes are required), or\nif the request has invalid credentials (`auth-failed`). It does *not* imply\nauthorization (i.e. scope satisfaction not guaranteed).", "clientId": "The clientId that made the request, if there was one.", "duration": "The duration in ms of the call.", "expires": "The expiration date of the credentials, if the header was authenticated.", "method": "The http method of the request.", "name": "The name of the API method", "public": "True if the endpoint requires no scopes.", "query": "Query params that were passed to the request (only ones that are specified as possible in the entry)", "resource": "The path of the http request.", "satisfyingScopes": "The set of scopes posessed by the caller that were\nused to authorize this request, or [] if scopes were not required.\n\nThis set of scopes is intuitively the scopes the caller used to\nauthorize the call. More precisely, it is the minimal subset of the supplied\nscopes required to satisfy the API method's authorization requirements.\nBut is not quite minimal in one sense: If several alternatives of an\n`AnyOf` in the API method's scope ression are satisfied, then the\nscopes used to satisfy *all* such alternatives are included.", "sourceIp": "The API method caller's IP", "statusCode": "The http status code that the endpoint resolved with." }, "level": "notice", "name": "apiMethod", "title": "API Method Report", "type": "monitor.apiMethod", "version": 2 }, { "description": "Audit event that happens .. .", "fields": { "action": "Type of action: create, edit, delete, etc..", "clientId": "ClientID associated with the change", "entity": "Name of the entity", "entityId": "Id of the entity", "service": "The name of the service." }, "level": "notice", "name": "auditEvent", "title": "Audit Event", "type": "audit-event", "version": 1 }, { "description": "A timer for calls to aws services.", "fields": { "duration": "How long it took to resolve the request in ms.", "operation": "The specific operation of this call.", "region": "The AWS region this service is in.", "service": "The name of the aws service being accessed." }, "level": "info", "name": "awsTimer", "title": "AWS Request Report", "type": "monitor.aws", "version": 1 }, { "description": "Azure VM instanceView returned 404 repeatedly for the same worker.\nWorker Manager treats this as missing to avoid getting stuck in a transient loop.", "fields": { "instanceView404Streak": "Consecutive count of instanceView 404 responses", "providerId": "Provider ID", "provisioningState": "Provisioning state returned by virtualMachines.get", "vmName": "Azure VM name", "workerGroup": "Worker Group", "workerId": "Worker ID", "workerPoolId": "Worker Pool ID" }, "level": "warning", "name": "azureInstanceViewRepeated404", "title": "Azure InstanceView Repeated 404", "type": "azure-instance-view-repeated-404", "version": 1 }, { "description": "When ARM template is being deployed with custom resource group name,\nAzure provider would create or update the resource group.\nThis is to make sure that deployment is run in the existing resource group.", "fields": { "location": "Location", "resourceGroupName": "Resource Group Name", "workerPoolId": "Worker Pool ID" }, "level": "notice", "name": "azureResourceGroupEnsured", "title": "Azure Resource Group Create or Update Information", "type": "azure-resource-group-ensure", "version": 1 }, { "description": "Records how an Azure worker's resources were torn down: 'fast' when the VM's\nNIC/IP/disks cascade-delete with the VM (so the per-resource GET/delete walk\nis skipped), or 'slow' for the resource-by-resource walk. Lets us measure the\nAPI-call reduction and confirm the fast path engages on cascading ARM pools.", "fields": { "launchConfigId": "Launch Config ID", "mode": "Teardown mode: 'fast' (cascade short-circuit) or 'slow' (full walk)", "providerId": "Provider ID", "workerId": "Worker ID", "workerPoolId": "Worker Pool ID" }, "level": "info", "name": "azureTeardownMode", "title": "Azure Worker Teardown Mode", "type": "azure-teardown-mode", "version": 1 }, { "description": "Azure ARM API returned HTTP 429 (Too Many Requests).\nIncludes rate-limit headers from the throttled response for diagnosing\nwhich subscription limits are being hit.", "fields": { "operationType": "HTTP operation category: read, write, or delete", "providerId": "Provider ID", "remainingDeletes": "x-ms-ratelimit-remaining-subscription-deletes value, or null", "remainingReads": "x-ms-ratelimit-remaining-subscription-reads value, or null", "remainingResource": "x-ms-ratelimit-remaining-resource raw value, or null", "remainingWrites": "x-ms-ratelimit-remaining-subscription-writes value, or null", "retryAfterSeconds": "Retry-After header value in seconds, or null if absent" }, "level": "warning", "name": "azureThrottled", "title": "Azure API Throttled (429)", "type": "azure-throttled", "version": 1 }, { "description": "The most basic timer.", "fields": { "duration": "The duration in ms of whatever was timed.", "key": "A key that should be unique to the logger prefix." }, "level": "info", "name": "basicTimer", "title": "Basic Timer", "type": "monitor.timer", "version": 1 }, { "description": "Metrics for cloud api calls", "fields": { "avg": "Average API call duration in milliseconds", "byStatus": "Map of HTTP status codes to counts", "failed": "Number of failed API calls", "max": "Maximum API call duration in milliseconds", "median": "Median API call duration in milliseconds", "min": "Minimum API call duration in milliseconds", "p95": "95th percentile API call duration in milliseconds", "p99": "99th percentile API call duration in milliseconds", "providerId": "Metrics for the given provider", "retries": "Number of retried API calls", "success": "Number of successful API calls", "total": "Total number of API calls made" }, "level": "notice", "name": "cloudApiMetrics", "title": "Cloud API call metrics", "type": "cloud-api-metrics", "version": 1 }, { "description": "Rate limiting engaged for a cloud api", "fields": { "duration": "Length of time the queue is paused for in ms.", "providerId": "Which provider has hit a limit", "queueName": "Which queue is paused -- there is one for each class of api request", "queueSize": "Number of requests remaining in the queue when it is paused.", "reason": "Either `errors` or `rateLimit`." }, "level": "any", "name": "cloudApiPaused", "title": "Cloud API Paused", "type": "cloud-api-paused", "version": 1 }, { "description": "A provider has resumed api requests.", "fields": { "providerId": "Which provider has hit a limit (each provider manages a single project)", "queueName": "Which queue is paused -- there is one for each class of api request" }, "level": "notice", "name": "cloudApiResumed", "title": "Cloud API Resumed", "type": "cloud-api-resumed", "version": 1 }, { "description": "A simple count. The reported values should be summed up in a time span.", "fields": { "key": "An arbitrary key.", "val": "A number that should be summed over a time span." }, "level": "info", "name": "countMetric", "title": "Generic Count", "type": "monitor.count", "version": 1 }, { "description": "This message is logged for each invocation of a Postgres stored function.", "fields": { "name": "The name of the DB function" }, "level": "info", "name": "dbFunctionCall", "title": "DB Method Call", "type": "db-function-call", "version": 1 }, { "description": "This message is logged every minute from each process that is using the database,\nonce for each pool. Most services have both a \"read\" pool and a \"write\" pool. The\nfields come from the node-postgres package.", "fields": { "idleCount": "The number of connections (out of the total) which are not currently in use", "pool": "The name of the pool within the process", "totalCount": "The total number of connections", "waitingCount": "The number of operations waiting for an idle connection" }, "level": "notice", "name": "dbPoolCounts", "title": "DB Pool Counts", "type": "db-pool-counts", "version": 1 }, { "description": "A specifically formatted error report. Will have at least the following but can contain more.", "fields": { "message": "Whatever message the error contained.", "name": "The name of the error.", "stack": "A nodejs stack trace." }, "level": "any", "name": "errorReport", "title": "Error Report", "type": "monitor.error", "version": 1 }, { "description": "An internal type for logging simple messages. No required fields.", "fields": {}, "level": "any", "name": "generic", "title": "Generic Message", "type": "monitor.generic", "version": 1 }, { "description": "A basic timer applied to a function.", "fields": { "duration": "The duration in ms of the function.", "name": "The name of the handler.", "status": "Whether or not the handler resolved successfully." }, "level": "info", "name": "handlerTimer", "title": "Handler Timer", "type": "monitor.timedHandler", "version": 1 }, { "description": "During worker pool provisioning, launch config selector may change config weight\nbased on current state of the system and initial data.\nThis event may help to understand how launch configs were used at selection time\nand what their adjusted weights were.", "fields": { "remainingCapacity": "An object with launchConfigId as a key and remaining capacity", "weights": "An object with launchConfigId as a key and adjusted weight as value", "workerPoolId": "Worker Pool ID" }, "level": "debug", "name": "launchConfigSelectorsDebug", "title": "Launch Config Selector Debug Information", "type": "launch-config-selector-debug", "version": 1 }, { "description": "An internal type for reporting malformed logging calls.", "fields": {}, "level": "err", "name": "loggingError", "title": "Error in Logging", "type": "monitor.loggingError", "version": 1 }, { "description": "A simple measurement. The reported values are expected to have statistics calculated over them.", "fields": { "key": "An arbitrary key.", "val": "A number that should be calculated over in a time span." }, "level": "info", "name": "measureMetric", "title": "Generic Measure", "type": "monitor.measure", "version": 1 }, { "description": "Status and duration of a periodic background process.", "fields": { "duration": "The duration in ms of the process.", "name": "The name of this process.", "status": "Either success, or exception depending on the outcome." }, "level": "any", "name": "periodic", "title": "Periodic Process", "type": "monitor.periodic", "version": 1 }, { "description": "A connection to Pulse has been established. Taskcluster services\nautomatically reconnect to Pulse periodically and in the event of\nan error, but rapid reconnections may signal a Pulse failure.", "fields": {}, "level": "info", "name": "pulseConnected", "title": "Connection to pulse established", "type": "pulse.connected", "version": 1 }, { "description": "A connection to Pulse has ended or failed. This may be due to normal\nreconnection or to an error communicating with Pulse. The service\nautomatically reconnects, so even an error is not necessarily a problem.", "fields": { "error": "The error message, if the disconnection was due to an error." }, "level": "info", "name": "pulseDisconnected", "title": "Connection to pulse has ended", "type": "pulse.disconnected", "version": 1 }, { "description": "A pulse publisher has been notified by the pulse server\nthat it has been blocked, usually because the server is\nresource-constrained. If `blocked` is false, then the\nnotification is that the publisher has been unblocked\nafter having previously been blocked.", "fields": { "blocked": "If true, the publisher has been blocked; if false, it has been unblocked." }, "level": "warning", "name": "pulsePublisherBlocked", "title": "Pulse server has (un)blocked a publisher", "type": "pulsePublisherBlocked", "version": 1 }, { "description": "Something has tried to register as a worker but failed. This could indicate either a bug\nor that somebody is trying to impersonate a worker.", "fields": { "error": "Error message from cloud that triggered this", "message": "Description of this error from the taskcluster side", "providerId": "The provider that did the work for this worker pool.", "workerId": "The worker that failed", "workerPoolId": "The worker pool ID" }, "level": "warning", "name": "registrationErrorWarning", "title": "Registration Error Warning", "type": "registration-error-warning", "version": 1 }, { "description": "A worker's identify proof message was signed by an unknown intermediate certificate.\nThis was successfully downloaded, verified by root CAs, and added to the CA store.", "fields": { "fingerprint": "The fingerprint of the certificate", "issuer": "The distinguished name of the certificate issuer", "subject": "The distinguished name of the certificate subject", "url": "The URL from which the certificate was downloaded" }, "level": "warning", "name": "registrationNewIntermediateCertificate", "title": "Registration of New Intermediate Certificate", "type": "registration-new-intermediate-certificate", "version": 1 }, { "description": "A worker registration attempted to download an intermediate certificate from\na URL that is outside the configured AIA allowlist.", "fields": { "providerId": "The provider that did the work for this worker pool.", "url": "The rejected intermediate certificate URL", "workerId": "The worker that failed", "workerPoolId": "The worker pool ID" }, "level": "warning", "name": "registrationRejectedIntermediateCertificateUrl", "title": "Registration Rejected Intermediate Certificate URL", "type": "registration-rejected-intermediate-certificate-url", "version": 1 }, { "description": "Metrics of a node process.", "fields": { "lastCpuUsage": "The output of node's process.cpuUsage", "lastMemoryUsage": "The output of node's process.memoryUsage" }, "level": "info", "name": "resourceMetrics", "title": "Resource Usage Metrics", "type": "monitor.resources", "version": 1 }, { "description": "The results of a worker-scanner run", "fields": { "providerId": "The provider that has completed this scan", "seen": "A map of workerPoolId that were seen to how much capacity was seen", "total": "Total number of workers seen" }, "level": "notice", "name": "scanSeen", "title": "Scan Seen", "type": "scan-seen", "version": 3 }, { "description": "The simple estimator has decided that we need some number of instances.", "fields": { "desiredCapacity": "Amount of capacity that this estimator thinks we should have", "existingCapacity": "Amount of currently requested and available capacity", "maxCapacity": "The maximum amount of capacity that should be running", "minCapacity": "The minimum amount of capacity that should be running", "pendingTasks": "The number of tasks the queue reports are pending for this worker pool", "requestedCapacity": "Amount of capacity that this estimator thinks we should add", "scalingRatio": "The ratio of workers to spawn per pending task for this worker pool.", "stoppingCapacity": "Amount of capacity being stopped", "workerPoolId": "The worker pool name" }, "level": "any", "name": "simpleEstimate", "title": "Simple Estimate Provided", "type": "simple-estimate", "version": 4 }, { "description": "A simple timekeeper for measuring arbitrary time spans.", "fields": { "duration": "The duration in ms of whatever was timed.", "key": "A key that should be unique to the logger prefix." }, "level": "info", "name": "timekeeper", "title": "TimeKeeper Report", "type": "monitor.timekeeper", "version": 1 }, { "description": "An error was reported regarding the given worker pool. Such errors are generally\nthe responsibility of the owner of the worker pool, but may also indicate issues\nwith the Taskcluster deployment and as such are reported in the service logs as\nwell. Note that the 'extra' data associated with such a report is not included\nhere. To see that, use the UI or API to view the worker pool errors directly.", "fields": { "description": "Description of the error", "errorId": "The unique id of this error report", "kind": "The error kind", "title": "The error title", "workerPoolId": "The workerPool where the error occurred" }, "level": "notice", "name": "workerError", "title": "Worker Error", "type": "worker-error", "version": 1 }, { "description": "A worker pool's provisioning run has completed", "fields": { "duration": "Time taken to provision the worker pool in ms", "providerId": "The provider that did the work for this worker pool.", "workerPoolId": "The worker pool ID" }, "level": "info", "name": "workerPoolProvisioned", "title": "Worker Pool Provisioned", "type": "worker-pool-provisioned", "version": 2 }, { "description": "A request has been made to stop a worker. This operation can sometimes\ntake some time.", "fields": { "providerId": "The provider that did the work for this worker pool.", "reason": "The reason this worker is being removed", "runningDuration": "Time in seconds since worker registered (null if never registered)", "workerAge": "Total time in seconds since worker was created", "workerId": "The worker that is being removed", "workerPoolId": "The worker pool ID" }, "level": "notice", "name": "workerRemoved", "title": "Worker Removed", "type": "worker-removed", "version": 2 }, { "description": "A worker has been requested from a cloud api", "fields": { "providerId": "The provider that did the work for this worker pool.", "terminateAfter": "Time after which worker should be terminated", "workerGroup": "The worker group for this worker", "workerId": "The worker that was created", "workerPoolId": "The worker pool ID" }, "level": "notice", "name": "workerRequested", "title": "Worker Requested", "type": "worker-requested", "version": 1 }, { "description": "A worker has been marked as running", "fields": { "providerId": "The provider that did the work for this worker pool.", "registrationDuration": "Time in seconds from worker creation to registration", "workerId": "The worker that is running", "workerPoolId": "The worker pool ID" }, "level": "notice", "name": "workerRunning", "title": "Worker Running", "type": "worker-running", "version": 2 }, { "description": "A worker has been marked as stopped", "fields": { "providerId": "The provider that did the work for this worker pool.", "runningDuration": "Time in seconds since worker registered (null if never registered)", "workerAge": "Total time in seconds since worker was created", "workerId": "The worker that was stopped", "workerPoolId": "The worker pool ID" }, "level": "notice", "name": "workerStopped", "title": "Worker Stopped", "type": "worker-stopped", "version": 2 } ] }