Similar to this post for which I see no resolution anywhere, I have an issue with the Cloudflare logpull integration erroring out about the 168h maximum:
"message":"Error while processing http request: failed to execute rf.collectResponse: failed to execute http client.Do: server responded with status code 400: bad query: error parsing time: invalid time range: too early: logs older than 168h0m0s are not available"
All settings within the integration are at or below the 24h setting, even. I see nowhere that I can adjust a time setting for the period for which to gather that is having ANY effect on this error, and it appears that no logs are coming in. A manual curl against the API, with a date range of 1 hour is having no issues, so I'd assume it's something either hard-coded or 'stuck' from a previous config. This was working until a few weeks ago, with no issue, but now I can't seem to work around it.
Our policy (with relevant ID's and Key's edited out):
{
"item": {
"id": "f5e823df-87d9-498f-bc49-3bc8fb08b5b4",
"version": "WzY4MDg5NDQsNF0=",
"name": "cloudflare-1",
"namespace": "default",
"description": "",
"package": {
"name": "cloudflare",
"title": "Cloudflare",
"version": "2.3.0"
},
"enabled": true,
"policy_id": "142374b0-368d-11ed-8fc9-9b05019573b8",
"inputs": [
{
"policy_template": "cloudflare",
"streams": [
{
"compiled_stream": {
"cursor": {
"last_timestamp": {
"fail_on_template_error": true,
"value": "[[.first_event.when]]"
}
},
"response.pagination": [
{
"set": {
"fail_on_template_error": true,
"value": "[[if (ne (len .last_response.body.result) 0)]][[add .last_response.page 1]][[end]]",
"target": "url.params.page"
}
}
],
"response.split": {
"target": "body.result"
},
"request.method": "GET",
"interval": "1h",
"request.url": "https://api.cloudflare.com/client/v4/accounts/<our account id>/audit_logs?page=1&direction=desc",
"processors": [
{
"add_fields": {
"fields": {
"account_id": "<our account id>"
},
"target": "_config"
}
}
],
"config_version": "2",
"request.timeout": "60s",
"request.transforms": [
{
"set": {
"value": "<our user>",
"target": "header.X-Auth-Email"
}
},
{
"set": {
"value": "<our auth key>",
"target": "header.X-Auth-Key"
}
},
{
"set": {
"default": """[[formatDate (now (parseDuration "-24h"))]]""",
"value": "[[.cursor.last_timestamp]]",
"target": "url.params.since"
}
}
],
"tags": [
"forwarded",
"cloudflare-audit",
"preserve_original_event"
],
"publisher_pipeline.disable_host": true
},
"data_stream": {
"type": "logs",
"dataset": "cloudflare.audit"
},
"vars": {
"initial_interval": {
"type": "text",
"value": "24h"
},
"interval": {
"type": "text",
"value": "1h"
},
"auth_key": {
"type": "password",
"value": "<our auth key>"
},
"processors": {
"type": "yaml"
},
"auth_email": {
"type": "text",
"value": "<our user>"
},
"account": {
"type": "text",
"value": "<our account id>"
},
"preserve_original_event": {
"type": "bool",
"value": true
},
"tags": {
"type": "text",
"value": [
"forwarded",
"cloudflare-audit"
]
}
},
"id": "httpjson-cloudflare.audit-f5e823df-87d9-498f-bc49-3bc8fb08b5b4",
"enabled": true
},
{
"compiled_stream": {
"cursor": {
"last_execution_datetime": {
"value": """[[.last_response.url.params.Get "end"]]"""
}
},
"response.decode_as": "application/x-ndjson",
"request.method": "GET",
"interval": "5m",
"request.url": "https://api.cloudflare.com/client/v4/zones/<our zone id>/logs/received",
"config_version": "2",
"request.timeout": "60s",
"request.transforms": [
{
"set": {
"value": "<our user>",
"target": "header.X-Auth-Email"
}
},
{
"set": {
"value": "<our auth key>",
"target": "header.X-Auth-Key"
}
},
{
"set": {
"default": """[[formatDate (((now).Add (parseDuration "-1m")).Add (parseDuration "-5m"))]]""",
"value": "[[.cursor.last_execution_datetime]]",
"target": "url.params.start"
}
},
{
"set": {
"default": """[[formatDate ((now).Add (parseDuration "-1m"))]]""",
"value": """[[formatDate ((parseDate .cursor.last_execution_datetime).Add (parseDuration "5m"))]]""",
"target": "url.params.end"
}
},
{
"set": {
"value": "CacheCacheStatus,CacheResponseBytes,CacheResponseStatus,CacheTieredFill,ClientASN,ClientCountry,ClientDeviceType,ClientIP,ClientIPClass,ClientRequestBytes,ClientRequestHost,ClientRequestMethod,ClientRequestPath,ClientRequestProtocol,ClientRequestReferer,ClientRequestURI,ClientRequestUserAgent,ClientSSLCipher,ClientSSLProtocol,ClientSrcPort,ClientXRequestedWith,EdgeColoCode,EdgeColoID,EdgeEndTimestamp,EdgePathingOp,EdgePathingSrc,EdgePathingStatus,EdgeRateLimitAction,EdgeRateLimitID,EdgeRequestHost,EdgeResponseBytes,EdgeResponseCompressionRatio,EdgeResponseContentType,EdgeResponseStatus,EdgeServerIP,EdgeStartTimestamp,FirewallMatchesActions,FirewallMatchesRuleIDs,FirewallMatchesSources,OriginIP,OriginResponseBytes,OriginResponseHTTPExpires,OriginResponseHTTPLastModified,OriginResponseStatus,OriginResponseTime,OriginSSLProtocol,ParentRayID,RayID,SecurityLevel,WAFAction,WAFFlags,WAFMatchedVar,WAFProfile,WAFRuleID,WAFRuleMessage,WorkerCPUTime,WorkerStatus,WorkerSubrequest,WorkerSubrequestCount,ZoneID,Action",
"target": "url.params.fields"
}
}
],
"tags": [
"forwarded",
"cloudflare-logpull",
"preserve_original_event"
],
"publisher_pipeline.disable_host": true
},
"data_stream": {
"type": "logs",
"dataset": "cloudflare.logpull"
},
"vars": {
"zone_id": {
"type": "text",
"value": "<our zone id>"
},
"interval": {
"type": "text",
"value": "5m"
},
"auth_key": {
"type": "password",
"value": "<our auth key>"
},
"processors": {
"type": "yaml"
},
"auth_email": {
"type": "text",
"value": "<our user>"
},
"auth_token": {
"type": "password",
"value": ""
},
"preserve_original_event": {
"type": "bool",
"value": true
},
"tags": {
"type": "text",
"value": [
"forwarded",
"cloudflare-logpull"
]
}
},
"id": "httpjson-cloudflare.logpull-f5e823df-87d9-498f-bc49-3bc8fb08b5b4",
"enabled": true
}
],
"vars": {
"api_url": {
"type": "text",
"value": "https://api.cloudflare.com"
},
"proxy_url": {
"type": "text"
},
"ssl": {
"type": "yaml"
},
"http_client_timeout": {
"type": "text",
"value": "60s"
}
},
"type": "httpjson",
"enabled": true
}
],
"revision": 31,
"created_at": "2022-09-28T15:33:55.018Z",
"created_by": "configuser",
"updated_at": "2022-11-29T18:30:12.769Z",
"updated_by": "configuser"
}
}
