fluent-12232020.conf.rt

## Prometheus Input Plugin Configuration

# input plugin that exports metrics
<source>
  @type prometheus
</source>

<source>
  @type monitor_agent
</source>

<source>
  @type forward
</source>

# input plugin that collects metrics from MonitorAgent
<source>
  @type prometheus_monitor
  <labels>
    host ${hostname}
  </labels>
</source>

# input plugin that collects metrics for output plugin
<source>
  @type prometheus_output_monitor
  <labels>
    host ${hostname}
  </labels>
</source>

# input plugin that collects metrics for in_tail plugin
<source>
  @type prometheus_tail_monitor
  <labels>
    host ${hostname}
  </labels>
</source>

<source>
  @type tail
  @id access_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log.pos"
  tag jfrog.rt.access.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id artifactory_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log.pos"
  tag jfrog.rt.artifactory.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id frontend_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos"
  tag jfrog.rt.frontend.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log.pos"
  tag jfrog.rt.metadata.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos"
  tag jfrog.rt.router.service
  <parse>
    @type none
  </parse>
</source>
# Strip out color codes then field extract the service fields
<filter jfrog.rt.**.service>
  @type record_transformer
  enable_ruby true
  <record>
    message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
  </record>
</filter>
<filter jfrog.rt.**.service>
  @type parser
  key_name message
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
      time_key timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</filter>
# End Service Fields Extraction
<source>
  @type tail
  @id router_traefik_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos"
  tag jfrog.rt.router.traefik
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] - (?<message>.+)$/
      time_key timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>

<source>
  @type tail
  @id access_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log.pos"
  tag jfrog.rt.access.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id artifactory_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log.pos"
  tag jfrog.rt.artifactory.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
    types response_content_length:integer, request_content_length:integer, return_status_code:integer
  </parse>
</source>
<filter jfrog.rt.artifactory.request>
  @type record_transformer
  enable_ruby true
  <record>
    user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")}
    repo ${record["request_url"].strip().start_with?("/api/download") ? (record["request_url"].strip().split('/')[3]) : ("")}
    artifact ${record["request_url"].strip().start_with?("/api/download") ? (val = record["request_url"].strip().split('/'); val[val.length()-1]) : ("")}
    dockerRepo ${record["request_url"].strip().include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") ? (record["request_url"].strip().split('/')[3]) : ("")}
    dockerImage ${record["request_url"].strip().include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2")  ? (record["request_url"].strip().split('/')[5]) : ("")}
    data_download ${record["response_content_length"] == -1 ? 0 : record["response_content_length"]}
    data_upload ${record["request_content_length"] == -1 ? 0 : record["request_content_length"]}
  </record>
</filter>
<source>
  @type tail
  @id frontend_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log.pos"
  tag jfrog.rt.frontend.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<filter jfrog.rt.frontend.request>
  @type record_transformer
  enable_ruby true
  <record>
    user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")}
  </record>
</filter>
<source>
  @type tail
  @id metadata_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log.pos"
  tag jfrog.rt.metadata.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<filter jfrog.rt.metadata.request>
  @type record_transformer
  enable_ruby true
  <record>
    user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")}
  </record>
</filter>
<source>
  @type tail
  @id router_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
  tag jfrog.rt.router.request
  <parse>
    @type json
    time_key time
    time_format %Y-%m-%dT%H:%M:%SZ
  </parse>
</source>
<source>
  @type tail
  @id artifactory_access_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log.pos"
  tag jfrog.rt.artifactory.access
  <parse>
    @type regexp
    expression /^(?<timestamp>[^ ]*) \[(?<trace_id>[^\]]*)\] \[(?<action_response>[^\]]*)\] (?<repo_path>.*) for client : (?<username>.+)/(?<ip>.+).$/
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id access_security_audit_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log.pos"
  tag jfrog.rt.access.audit
  <parse>
    @type regexp
    expression /^(?<timestamp>[^ ]*)\|(?<token_id>[^ ]*)\|(?<user_ip>[^ ]*)\|(?<user>[^ ]*)\|(?<logged_principal>[^ ]*)\|(?<entity_name>[^ ]*)\|(?<event_type>[^ ]*)\|(?<event>[^ ]*)\|(?<data_changed>.*)/
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>

# WHAT LOG IT WAS INTO THE JSON
<filter jfrog.**>
  @type record_transformer
  <record>
    hostname "#{Socket.gethostname}"
    log_source ${tag}
  </record>
</filter>

#JFROG CALL HOME
<source>
  @type exec
  tag jfrog.callhome
  command "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/fluentd-1.11.0-linux-x86_64/lib/ruby/bin/gem list --local | grep fluent | sed 's/ (/:/g' | sed 's/)//g'  | sed ':a;N;$!ba;s/\n/\t/g'"
  run_interval 1d
  <parse>
    @type ltsv
  </parse>
</source>
<source>
  @type exec
  tag jfrog.callhome
  command /opt/td-agent/embedded/bin/gem list --local | grep fluent | sed 's/ (/:/g' | sed 's/)//g'  | sed ':a;N;$!ba;s/\n/\t/g'
  run_interval 1d
  <parse>
    @type ltsv
  </parse>
</source>
<filter jfrog.callhome>
  @type record_transformer
  renew_record true
  keep_keys 'productId,features'
  enable_ruby true
  <record>
    productId 'jfrogLogAnalytics/v0.1.0'
    features ${return(record.map { |k,v| { "featureId" => (k + ':' + v).to_sym} })}
  </record>
</filter>
<match jfrog.callhome>
  @type http
  endpoint http://localhost:8081/artifactory/api/system/usage
  open_timeout 5
  content_type application/json
  <format>
    @type json
  </format>
  <buffer>
    flush_interval 5s
  </buffer>
</match>
#END CALL HOME

<filter jfrog.rt.artifactory.request>
  @type prometheus

  <metric>
    name jfrog_rt_data_download
    type counter
    desc artifactory data download
    key data_download
    <labels>
      data_download ${data_download}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_download_repo
    type counter
    desc artifactory data download by repo
    key data_download
    <labels>
      repo ${repo}
      data_download ${data_download}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_download_by_docker_repo
    type counter
    desc artifactory data download by docker repo
    key data_download
    <labels>
      dockerRepo ${dockerRepo}
      data_download ${data_download}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_download_by_remote_address
    type counter
    desc artifactory data download by remote address
    key data_download
    <labels>
      remote_address ${remote_address}
      data_download ${data_download}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_upload
    type counter
    desc artifactory data upload
    key data_upload
    <labels>
      data_upload ${data_upload}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_upload_by_repo
    type counter
    desc artifactory data upload by repo
    key data_upload
    <labels>
      repo ${repo}
      data_upload ${data_upload}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_upload_by_docker_repo
    type counter
    desc artifactory data upload by docker repo
    key data_upload
    <labels>
      dockerRepo ${dockerRepo}
      data_upload ${data_upload}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_upload_by_remote_address
    type counter
    desc artifactory data upload by remote address
    key data_upload
    <labels>
      remote_address ${remote_address}
      data_upload ${data_upload}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_req
    type counter
    desc artifactory requests
    <labels>
      return_status ${return_status}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_req_by_artifact
    type counter
    desc artifactory requests by artifact
    <labels>
      artifact ${artifact}
      return_status ${return_status}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_req_by_repo
    type counter
    desc artifactory requests by repo
    <labels>
      repo ${repo}
      return_status ${return_status}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_req_by_docker_image
    type counter
    desc artifactory requests by docker image
    <labels>
      dockerImage ${dockerImage}
      return_status ${return_status}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_req_by_docker_repo
    type counter
    desc artifactory requests by docker repo
    <labels>
      dockerRepo ${dockerRepo}
      return_status ${return_status}
    </labels>
  </metric>

</filter>

<filter jfrog.rt.artifactory.service>
@type prometheus
  <metric>
    name jfrog_rt_log_level
    type counter
    desc artifactory log_levels
    <labels>
      log_level ${log_level}
    </labels>
  </metric>

  <metric>
      name jfrog_rt_service_message
      type counter
      desc artifactory service message
      <labels>
        message ${message}
      </labels>
    </metric>
</filter>

<filter jfrog.rt.artifactory.access>
@type prometheus

<metric>
  name jfrog_rt_access
  type counter
  desc artifactory access
  <labels>
    username ${username}
    repo_path ${repo_path}
    action_response ${action_response}
    ip ${ip}
  </labels>
</metric>

</filter>

<filter jfrog.rt.access.audit>
@type prometheus

<metric>
  name jfrog_rt_access_audit
  type counter
  desc artifactory access audit
  <labels>
    user ${user}
  </labels>
</metric>

</filter>