From 569fb6aa1e4d6e5b7cf03f0236fa070183611479 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Sun, 17 Mar 2024 21:25:26 +0100 Subject: [PATCH 1/2] ignore user-generated docs files and remove already committed files from the tree --- .gitignore | 4 +- docs/user/event.md | 646 ---------- docs/user/feeds.md | 2900 -------------------------------------------- 3 files changed, 3 insertions(+), 3547 deletions(-) delete mode 100644 docs/user/event.md delete mode 100644 docs/user/feeds.md diff --git a/.gitignore b/.gitignore index 038ad329d..4df166a66 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,9 @@ venv/ .env # mkdocs -docs_build +docs_build/ +docs/user/feeds.md +docs/user/event.md # Debian build filed debian/files diff --git a/docs/user/event.md b/docs/user/event.md deleted file mode 100644 index 796440c63..000000000 --- a/docs/user/event.md +++ /dev/null @@ -1,646 +0,0 @@ - - -# Event - -An event represents individual piece of data processed by IntelMQ. It uses JSON format. - -Example Event: - -```json -{ - "source.geolocation.cc": "JO", - "malware.name": "qakbot", - "source.ip": "82.212.115.188", - "source.asn": 47887, - "classification.type": "c2-server", - "extra.status": "offline", - "source.port": 443, - "classification.taxonomy": "malicious-code", - "source.geolocation.latitude": 31.9522, - "feed.accuracy": 100, - "extra.last_online": "2023-02-16", - "time.observation": "2023-02-16T09:55:12+00:00", - "source.geolocation.city": "amman", - "source.network": "82.212.115.0/24", - "time.source": "2023-02-15T14:19:09+00:00", - "source.as_name": "NEU-AS", - "source.geolocation.longitude": 35.939, - "feed.name": "abusech-feodo-c2-tracker" - } -``` - -## Minimum Requirements - -Below, we have enumerated the minimum recommended requirements for an actionable abuse event. These keys should be -present for the abuse report to make sense for the end recipient. Please note that if you choose to anonymize your -sources, you can substitute **feed.name** with **feed.code**. At least one of the fields **ip**, **fqdn**, **url** or **account** should be present. All the rest of the keys are optional. This list of required fields is *not* enforced by IntelMQ. - -| Field | Terminology | -| ----------------------- | ----------- | -| feed.name | Should | -| classification.type | Should | -| classification.taxonomy | Should | -| time.source | Should | -| time.observation | Should | -| source.ip | Should\* | -| source.fqdn | Should\* | -| source.url | Should\* | -| source.account | Should\* | - -\* at least one of them - -## Classification - -IntelMQ classifies events using three labels: `classification.taxonomy`, `classification.type` and `classification.identifier`. This tuple of three values can be used for deduplication of events and describes what happened. - -The taxonomy can be automatically added by the taxonomy expert bot based on the given type. The following classification scheme loosely follows the [Reference Security Incident Taxonomy (RSIT)](https://github.com/enisaeu/Reference-Security-Incident-Taxonomy-Task-Force/): - -| Classification Taxonomy | Classification Type | Description | -|----------------------|----------------------|----------------------| -| abusive-content | harmful-speech | Discreditation or discrimination of somebody, cyber stalking, racism or threats against one or more individuals. | -| abusive-content | spam | Or 'Unsolicited Bulk Email', this means that the recipient has not granted verifiable permission for the message to be sent and that the message is sent as part of a larger collection of messages, all having a functionally comparable content. | -| abusive-content | violence | Child pornography, glorification of violence, etc. | -| availability | ddos | Distributed Denial of Service attack, e.g. SYN-Flood or UDP-based reflection/amplification attacks. | -| availability | dos | Denial of Service attack, e.g. sending specially crafted requests to a web application which causes the application to crash or slow down. | -| availability | misconfiguration | Software misconfiguration resulting in service availability issues, e.g. DNS server with outdated DNSSEC Root Zone KSK. | -| availability | outage | Outage caused e.g. by air condition failure or natural disaster. | -| availability | sabotage | Physical sabotage, e.g cutting wires or malicious arson. | -| fraud | copyright | Offering or Installing copies of unlicensed commercial software or other copyright protected materials (Warez). | -| fraud | masquerade | Type of attack in which one entity illegitimately impersonates the identity of another in order to benefit from it. | -| fraud | phishing | Masquerading as another entity in order to persuade the user to reveal private credentials. | -| fraud | unauthorized-use-of-resources | Using resources for unauthorized purposes including profit-making ventures, e.g. the use of e-mail to participate in illegal profit chain letters or pyramid schemes. | -| information-content-security | data-leak | Leaked confidential information like credentials or personal data. | -| information-content-security | data-loss | Loss of data, e.g. caused by harddisk failure or physical theft. | -| information-content-security | unauthorised-information-access | Unauthorized access to information, e.g. by abusing stolen login credentials for a system or application, intercepting traffic or gaining access to physical documents. | -| information-content-security | unauthorised-information-modification | Unauthorised modification of information, e.g. by an attacker abusing stolen login credentials for a system or application or a ransomware encrypting data. | -| information-gathering | scanner | Attacks that send requests to a system to discover weaknesses. This also includes testing processes to gather information on hosts, services and accounts. Examples: fingerd, DNS querying, ICMP, SMTP (EXPN, RCPT, \...), port scanning. | -| information-gathering | sniffing | Observing and recording of network traffic (wiretapping). | -| information-gathering | social-engineering | Gathering information from a human being in a non-technical way (e.g. lies, tricks, bribes, or threats). This IOC refers to a resource, which has been observed to perform brute-force attacks over a given application protocol. | -| intrusion-attempts | brute-force | Multiple login attempts (Guessing/cracking of passwords, brute force). | -| intrusion-attempts | exploit | An attack using an unknown exploit. | -| intrusion-attempts | ids-alert | IOCs based on a sensor network. This is a generic IOC denomination, should it be difficult to reliably denote the exact type of activity involved for example due to an anecdotal nature of the rule that triggered the alert. | -| intrusions | application-compromise| Compromise of an application by exploiting (un)known software vulnerabilities, e.g. SQL injection. | -| intrusions | burglary | Physical intrusion, e.g. into corporate building or data center. | -| intrusions | privileged-account-compromise | Compromise of a system where the attacker gained administrative privileges. | -| intrusions | system-compromise | Compromise of a system, e.g. unauthorised logins or commands. This includes compromising attempts on honeypot systems. | -| intrusions | unprivileged-account-compromise | Compromise of a system using an unprivileged (user/service) account. | -| malicious-code | c2-server | This is a command and control server in charge of a given number of botnet drones. | -| malicious-code | infected-system | This is a compromised machine, which has been observed to make a connection to a command and control server. | -| malicious-code | malware-configuration | This is a resource which updates botnet drones with a new configuration. | -| malicious-code | malware-distribution | URI used for malware distribution, e.g. a download URL included in fake invoice malware spam. | -| other | blacklist | Some sources provide blacklists, which clearly refer to abusive behavior, such as spamming, but fail to denote the exact reason why a given identity has been blacklisted. The reason may be that the justification is anecdotal or missing entirely. This type should only be used if the typing fits the definition of a blacklist, but an event specific denomination is not possible for one reason or another. Not in RSIT. | -| other | dga-domain | DGA Domains are seen various families of malware that are used to periodically generate a large number of domain names that can be used as rendezvous points with their command and control servers. Not in RSIT. | -| other | other | All incidents which don't fit in one of the given categories should be put into this class. | -| other | malware | An IoC referring to a malware (sample) itself. Not in RSIT. | -| other | proxy | This refers to the use of proxies from inside your network. Not in RSIT. | -| test | test | Meant for testing. Not in RSIT. | -| other | tor | This IOC refers to incidents related to TOR network infrastructure. Not in RSIT. | -| other | undetermined | The categorisation of the incident is unknown/undetermined. | -| vulnerable | ddos-amplifier | Publicly accessible services that can be abused for conducting DDoS reflection/amplification attacks, e.g. DNS open-resolvers or NTP servers with monlist enabled. | -| vulnerable | information-disclosure | Publicly accessible services potentially disclosing sensitive information, e.g. SNMP or Redis. | -| vulnerable | potentially-unwanted-accessible | Potentially unwanted publicly accessible services, e.g. Telnet, RDP or VNC. | -| vulnerable | vulnerable-system | A system which is vulnerable to certain attacks. Example: misconfigured client proxy settings (example: WPAD), outdated operating system version, etc. | -| vulnerable | weak-crypto | Publicly accessible services offering weak crypto, e.g. web servers susceptible to POODLE/FREAK attacks. | - -## Meaning of source and destination identities - -Meaning of source and destination identities for each `classification.type` can be different. Usually the main information is in the `source.*` fields. - -The `classification.identifier` is often a normalized malware name, grouping many variants or the affected network protocol. - -Examples of the meaning of the *source* and *destination* fields for various `classification.type` and possible identifiers are shown here. - -| Classification Type | Source | Destination | Possible Identifiers | -| --------------------- | -------------------------------------- | -------------------- | ------------------------------------ | -| blacklist | blacklisted device | | | -| brute-force | attacker | target | | -| c2-server | (sinkholed) c&c server | | zeus, palevo, feodo | -| ddos | attacker | target | | -| dga-domain | infected device | | | -| dropzone | server hosting stolen data | | | -| exploit | hosting server | | | -| ids-alert | triggering device | | | -| infected-system | infected device | contacted c&c server | | -| malware | infected device | | zeus, palevo, feodo | -| malware-configuration | infected device | | | -| malware-distribution | server hosting malware | | | -| phishing | phishing website | | | -| proxy | server allowing policy/security bypass | | | -| scanner | scanning device | scanned device | http, modbus, wordpress | -| spam | infected device | targeted server | | -| system-compromise | server | | | -| vulnerable-system | vulnerable device | | heartbleed, openresolver, snmp, wpad | - -Examples: - -- If an event describes IP address that connects to a zeus command and control server, it's about the infected device. Therefore the `classification.taxonomy` is `malicious-code`, `classification.type` is `infected-system` and the `classification.identifier` is `zeus`. - -- If an event describes IP address where a command and control server is running, the event's -`classification.type` is `c2server`. The `malware.name` can have the full name, eg. `zeus_p2p`. - -## Additional Information - -Information that do not fit into any of the event fields should be placed in the `extra` namespace.Therefore the keys must be prefixed `extra.` string. There are no other rules on key names and values for additional information. - -## Fields Reference - -Here you can find detailed information about all the possible fields used in an event. - - -### `classification.identifier`
- -**Type:** [String](#string) - -The lowercase identifier defines the actual software or service (e.g. ``heartbleed`` or ``ntp_version``) or standardized malware name (e.g. ``zeus``). Note that you MAY overwrite this field during processing for your individual setup. This field is not standardized across IntelMQ setups/users. - -### `classification.taxonomy`
- -**Type:** [ClassificationTaxonomy](#classificationtaxonomy) - -We recognize the need for the CSIRT teams to apply a static (incident) taxonomy to abuse data. With this goal in mind the type IOC will serve as a basis for this activity. Each value of the dynamic type mapping translates to a an element in the static taxonomy. The European CSIRT teams for example have decided to apply the eCSIRT.net incident classification. The value of the taxonomy key is thus a derivative of the dynamic type above. For more information about check `ENISA taxonomies `_. - -### `classification.type`
- -**Type:** [ClassificationType](#classificationtype) - -The abuse type IOC is one of the most crucial pieces of information for any given abuse event. The main idea of dynamic typing is to keep our ontology flexible, since we need to evolve with the evolving threatscape of abuse data. In contrast with the static taxonomy below, the dynamic typing is used to perform business decisions in the abuse handling pipeline. Furthermore, the value data set should be kept as minimal as possible to avoid *type explosion*, which in turn dilutes the business value of the dynamic typing. In general, we normally have two types of abuse type IOC: ones referring to a compromised resource or ones referring to pieces of the criminal infrastructure, such as a command and control servers for example. - -### `comment`
- -**Type:** [String](#string) - -Free text commentary about the abuse event inserted by an analyst. - -### `destination.abuse_contact`
- -**Type:** [LowercaseString](#lowercasestring) - -Abuse contact for destination address. A comma separated list. - -### `destination.account`
- -**Type:** [String](#string) - -An account name or email address, which has been identified to relate to the destination of an abuse event. - -### `destination.allocated`
- -**Type:** [DateTime](#datetime) - -Allocation date corresponding to BGP prefix. - -### `destination.as_name`
- -**Type:** [String](#string) - -The autonomous system name to which the connection headed. - -### `destination.asn`
- -**Type:** [ASN](#asn) - -The autonomous system number to which the connection headed. - -### `destination.domain_suffix`
- -**Type:** [FQDN](#fqdn) - -The suffix of the domain from the public suffix list. - -### `destination.fqdn`
- -**Type:** [FQDN](#fqdn) - -A DNS name related to the host from which the connection originated. DNS allows even binary data in DNS, so we have to allow everything. A final point is stripped, string is converted to lower case characters. - -### `destination.geolocation.cc`
- -**Type:** [UppercaseString](#uppercasestring) - -Country-Code according to ISO3166-1 alpha-2 for the destination IP. - -### `destination.geolocation.city`
- -**Type:** [String](#string) - -Some geolocation services refer to city-level geolocation. - -### `destination.geolocation.country`
- -**Type:** [String](#string) - -The country name derived from the ISO3166 country code (assigned to cc field). - -### `destination.geolocation.latitude`
- -**Type:** [Float](#float) - -Latitude coordinates derived from a geolocation service, such as MaxMind geoip db. - -### `destination.geolocation.longitude`
- -**Type:** [Float](#float) - -Longitude coordinates derived from a geolocation service, such as MaxMind geoip db. - -### `destination.geolocation.region`
- -**Type:** [String](#string) - -Some geolocation services refer to region-level geolocation. - -### `destination.geolocation.state`
- -**Type:** [String](#string) - -Some geolocation services refer to state-level geolocation. - -### `destination.ip`
- -**Type:** [IPAddress](#ipaddress) - -The IP which is the target of the observed connections. - -### `destination.local_hostname`
- -**Type:** [String](#string) - -Some sources report an internal hostname within a NAT related to the name configured for a compromised system - -### `destination.local_ip`
- -**Type:** [IPAddress](#ipaddress) - -Some sources report an internal (NATed) IP address related a compromised system. N.B. RFC1918 IPs are OK here. - -### `destination.network`
- -**Type:** [IPNetwork](#ipnetwork) - -CIDR for an autonomous system. Also known as BGP prefix. If multiple values are possible, select the most specific. - -### `destination.port`
- -**Type:** [Integer](#integer) - -The port to which the connection headed. - -### `destination.registry`
- -**Type:** [Registry](#registry) - -The IP registry a given ip address is allocated by. - -### `destination.reverse_dns`
- -**Type:** [FQDN](#fqdn) - -Reverse DNS name acquired through a reverse DNS query on an IP address. N.B. Record types other than PTR records may also appear in the reverse DNS tree. Furthermore, unfortunately, there is no rule prohibiting people from writing anything in a PTR record. Even JavaScript will work. A final point is stripped, string is converted to lower case characters. - -### `destination.tor_node`
- -**Type:** [Boolean](#boolean) - -If the destination IP was a known tor node. - -### `destination.url`
- -**Type:** [URL](#url) - -A URL denotes on IOC, which refers to a malicious resource, whose interpretation is defined by the abuse type. A URL with the abuse type phishing refers to a phishing resource. - -### `destination.urlpath`
- -**Type:** [String](#string) - -The path portion of an HTTP or related network request. - -### `event_description.target`
- -**Type:** [String](#string) - -Some sources denominate the target (organization) of a an attack. - -### `event_description.text`
- -**Type:** [String](#string) - -A free-form textual description of an abuse event. - -### `event_description.url`
- -**Type:** [URL](#url) - -A description URL is a link to a further description of the the abuse event in question. - -### `event_hash`
- -**Type:** [UppercaseString](#uppercasestring) - -Computed event hash with specific keys and values that identify a unique event. At present, the hash should default to using the SHA1 function. Please note that for an event hash to be able to match more than one event (deduplication) the receiver of an event should calculate it based on a minimal set of keys and values present in the event. Using for example the observation time in the calculation will most likely render the checksum useless for deduplication purposes. - -### `extra`
- -**Type:** [JSONDict](#jsondict) - -All anecdotal information, which cannot be parsed into the data harmonization elements. E.g. os.name, os.version, etc. **Note**: this is only intended for mapping any fields which can not map naturally into the data harmonization. It is not intended for extending the data harmonization with your own fields. - -### `feed.accuracy`
- -**Type:** [Accuracy](#accuracy) - -A float between 0 and 100 that represents how accurate the data in the feed is - -### `feed.code`
- -**Type:** [String](#string) - -Code name for the feed, e.g. DFGS, HSDAG etc. - -### `feed.documentation`
- -**Type:** [String](#string) - -A URL or hint where to find the documentation of this feed. - -### `feed.name`
- -**Type:** [String](#string) - -Name for the feed, usually found in collector bot configuration. - -### `feed.provider`
- -**Type:** [String](#string) - -Name for the provider of the feed, usually found in collector bot configuration. - -### `feed.url`
- -**Type:** [URL](#url) - -The URL of a given abuse feed, where applicable - -### `malware.hash.md5`
- -**Type:** [String](#string) - -A string depicting an MD5 checksum for a file, be it a malware sample for example. - -### `malware.hash.sha1`
- -**Type:** [String](#string) - -A string depicting a SHA1 checksum for a file, be it a malware sample for example. - -### `malware.hash.sha256`
- -**Type:** [String](#string) - -A string depicting a SHA256 checksum for a file, be it a malware sample for example. - -### `malware.name`
- -**Type:** [LowercaseString](#lowercasestring) - -The malware name in lower case. - -### `malware.version`
- -**Type:** [String](#string) - -A version string for an identified artifact generation, e.g. a crime-ware kit. - -### `misp.attribute_uuid`
- -**Type:** [LowercaseString](#lowercasestring) - -MISP - Malware Information Sharing Platform & Threat Sharing UUID of an attribute. - -### `misp.event_uuid`
- -**Type:** [LowercaseString](#lowercasestring) - -MISP - Malware Information Sharing Platform & Threat Sharing UUID. - -### `output`
- -**Type:** [JSON](#json) - -Event data converted into foreign format, intended to be exported by output plugin. - -### `protocol.application`
- -**Type:** [LowercaseString](#lowercasestring) - -e.g. vnc, ssh, sip, irc, http or smtp. - -### `protocol.transport`
- -**Type:** [LowercaseString](#lowercasestring) - -e.g. tcp, udp, icmp. - -### `raw`
- -**Type:** [Base64](#base64) - -The original line of the event from encoded in base64. - -### `rtir_id`
- -**Type:** [Integer](#integer) - -Request Tracker Incident Response ticket id. - -### `screenshot_url`
- -**Type:** [URL](#url) - -Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation. - -### `source.abuse_contact`
- -**Type:** [LowercaseString](#lowercasestring) - -Abuse contact for source address. A comma separated list. - -### `source.account`
- -**Type:** [String](#string) - -An account name or email address, which has been identified to relate to the source of an abuse event. - -### `source.allocated`
- -**Type:** [DateTime](#datetime) - -Allocation date corresponding to BGP prefix. - -### `source.as_name`
- -**Type:** [String](#string) - -The autonomous system name from which the connection originated. - -### `source.asn`
- -**Type:** [ASN](#asn) - -The autonomous system number from which originated the connection. - -### `source.domain_suffix`
- -**Type:** [FQDN](#fqdn) - -The suffix of the domain from the public suffix list. - -### `source.fqdn`
- -**Type:** [FQDN](#fqdn) - -A DNS name related to the host from which the connection originated. DNS allows even binary data in DNS, so we have to allow everything. A final point is stripped, string is converted to lower case characters. - -### `source.geolocation.cc`
- -**Type:** [UppercaseString](#uppercasestring) - -Country-Code according to ISO3166-1 alpha-2 for the source IP. - -### `source.geolocation.city`
- -**Type:** [String](#string) - -Some geolocation services refer to city-level geolocation. - -### `source.geolocation.country`
- -**Type:** [String](#string) - -The country name derived from the ISO3166 country code (assigned to cc field). - -### `source.geolocation.cymru_cc`
- -**Type:** [UppercaseString](#uppercasestring) - -The country code denoted for the ip by the Team Cymru asn to ip mapping service. - -### `source.geolocation.geoip_cc`
- -**Type:** [UppercaseString](#uppercasestring) - -MaxMind Country Code (ISO3166-1 alpha-2). - -### `source.geolocation.latitude`
- -**Type:** [Float](#float) - -Latitude coordinates derived from a geolocation service, such as MaxMind geoip db. - -### `source.geolocation.longitude`
- -**Type:** [Float](#float) - -Longitude coordinates derived from a geolocation service, such as MaxMind geoip db. - -### `source.geolocation.region`
- -**Type:** [String](#string) - -Some geolocation services refer to region-level geolocation. - -### `source.geolocation.state`
- -**Type:** [String](#string) - -Some geolocation services refer to state-level geolocation. - -### `source.ip`
- -**Type:** [IPAddress](#ipaddress) - -The ip observed to initiate the connection - -### `source.local_hostname`
- -**Type:** [String](#string) - -Some sources report a internal hostname within a NAT related to the name configured for a compromised system - -### `source.local_ip`
- -**Type:** [IPAddress](#ipaddress) - -Some sources report a internal (NATed) IP address related a compromised system. N.B. RFC1918 IPs are OK here. - -### `source.network`
- -**Type:** [IPNetwork](#ipnetwork) - -CIDR for an autonomous system. Also known as BGP prefix. If multiple values are possible, select the most specific. - -### `source.port`
- -**Type:** [Integer](#integer) - -The port from which the connection originated. - -### `source.registry`
- -**Type:** [Registry](#registry) - -The IP registry a given ip address is allocated by. - -### `source.reverse_dns`
- -**Type:** [FQDN](#fqdn) - -Reverse DNS name acquired through a reverse DNS query on an IP address. N.B. Record types other than PTR records may also appear in the reverse DNS tree. Furthermore, unfortunately, there is no rule prohibiting people from writing anything in a PTR record. Even JavaScript will work. A final point is stripped, string is converted to lower case characters. - -### `source.tor_node`
- -**Type:** [Boolean](#boolean) - -If the source IP was a known tor node. - -### `source.url`
- -**Type:** [URL](#url) - -A URL denotes an IOC, which refers to a malicious resource, whose interpretation is defined by the abuse type. A URL with the abuse type phishing refers to a phishing resource. - -### `source.urlpath`
- -**Type:** [String](#string) - -The path portion of an HTTP or related network request. - -### `status`
- -**Type:** [String](#string) - -Status of the malicious resource (phishing, dropzone, etc), e.g. online, offline. - -### `time.observation`
- -**Type:** [DateTime](#datetime) - -The time the collector of the local instance processed (observed) the event. - -### `time.source`
- -**Type:** [DateTime](#datetime) - -The time of occurrence of the event as reported the feed (source). - -### `tlp`
- -**Type:** [TLP](#tlp) - -Traffic Light Protocol level of the event. - diff --git a/docs/user/feeds.md b/docs/user/feeds.md deleted file mode 100644 index 06c67fb46..000000000 --- a/docs/user/feeds.md +++ /dev/null @@ -1,2900 +0,0 @@ - - -# Feeds - -The available feeds are grouped by the provider of the feeds. -For each feed the collector and parser that can be used is documented as well as any feed-specific parameters. -To add feeds to this file add them to `intelmq/etc/feeds.yaml` and then rebuild the documentation. - -## Abuse.ch - -### Feodo Tracker - -List of botnet Command & Control servers (C&Cs) tracked by Feodo Tracker, associated with Dridex and Emotet (aka Heodo). - -**Public:** yes - -**Revision:** 2022-11-15 - -**Documentation:** - -**Additional Information:** The data in the column Last Online is used for `time.source` if available, with 00:00 as time. Otherwise first_seen is used as `time.source`. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://feodotracker.abuse.ch/downloads/ipblocklist.json - name: Feodo Tracker - provider: Abuse.ch - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.abusech.parser_feodotracker -``` - ---- - - -### URLhaus - -URLhaus is a project from abuse.ch with the goal of sharing malicious URLs that are being used for malware distribution. URLhaus offers a country, ASN (AS number) and Top Level Domain (TLD) feed for network operators / Internet Service Providers (ISPs), Computer Emergency Response Teams (CERTs) and domain registries. - -**Public:** yes - -**Revision:** 2020-07-07 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://urlhaus.abuse.ch/feeds/tld//, https://urlhaus.abuse.ch/feeds/country//, or https://urlhaus.abuse.ch/feeds/asn// - name: URLhaus - provider: Abuse.ch - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.generic.parser_csv -parameters: - columns: ["time.source", "source.url", "status", "classification.type|__IGNORE__", "source.fqdn|__IGNORE__", "source.ip", "source.asn", "source.geolocation.cc"] - default_url_protocol: http:// - delimiter: , - skip_header: False - type_translation: [{"malware_download": "malware-distribution"}] -``` - ---- - - -## AlienVault - -### OTX - -AlienVault OTX Collector is the bot responsible to get the report through the API. Report could vary according to subscriptions. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.alienvault_otx.collector -parameters: - api_key: {{ your API key }} - name: OTX - provider: AlienVault -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.alienvault.parser_otx -``` - ---- - - -### Reputation List - -List of malicious IPs. - -**Public:** yes - -**Revision:** 2018-01-20 - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://reputation.alienvault.com/reputation.data - name: Reputation List - provider: AlienVault - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.alienvault.parser -``` - ---- - - -## AnubisNetworks - -### Cyberfeed Stream - -Fetches and parsers the Cyberfeed data stream. - -**Public:** no - -**Revision:** 2020-06-15 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http_stream -parameters: - http_url: https://prod.cyberfeed.net/stream?key={{ your API key }} - name: Cyberfeed Stream - provider: AnubisNetworks - strip_lines: true -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.anubisnetworks.parser -parameters: - use_malware_familiy_as_classification_identifier: True -``` - ---- - - -## Bambenek - -### C2 Domains - -Master Feed of known, active and non-sinkholed C&Cs domain names. Requires access credentials. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - -**Additional Information:** License: https://osint.bambenekconsulting.com/license.txt - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_password: __PASSWORD__ - http_url: https://faf.bambenekconsulting.com/feeds/c2-dommasterlist.txt - http_username: __USERNAME__ - name: C2 Domains - provider: Bambenek - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.bambenek.parser -``` - ---- - - -### C2 IPs - -Master Feed of known, active and non-sinkholed C&Cs IP addresses. Requires access credentials. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - -**Additional Information:** License: https://osint.bambenekconsulting.com/license.txt - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_password: __PASSWORD__ - http_url: https://faf.bambenekconsulting.com/feeds/c2-ipmasterlist.txt - http_username: __USERNAME__ - name: C2 IPs - provider: Bambenek - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.bambenek.parser -``` - ---- - - -### DGA Domains - -Domain feed of known DGA domains from -2 to +3 days - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - -**Additional Information:** License: https://osint.bambenekconsulting.com/license.txt - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://faf.bambenekconsulting.com/feeds/dga-feed.txt - name: DGA Domains - provider: Bambenek - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.bambenek.parser -``` - ---- - - -## Benkow - -### Malware Panels Tracker - -Benkow Panels tracker is a list of fresh panel from various malware. The feed is available on the webpage: http://benkow.cc/passwords.php - -**Public:** yes - -**Revision:** 2022-11-16 - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://benkow.cc/export.php - name: Malware Panels Tracker - provider: Benkow -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.generic.parser_csv -parameters: - columns: ["__IGNORE__", "malware.name", "source.url", "source.fqdn|source.ip", "time.source"] - columns_required: [false, true, true, false, true] - defaults_fields: {'classification.type': 'c2-server'} - delimiter: ; - skip_header: True -``` - ---- - - -## Blocklist.de - -### Apache - -Blocklist.DE Apache Collector is the bot responsible to get the report from source of information. All IP addresses which have been reported within the last 48 hours as having run attacks on the service Apache, Apache-DDOS, RFI-Attacks. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/apache.txt - name: Apache - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### Bots - -Blocklist.DE Bots Collector is the bot responsible to get the report from source of information. All IP addresses which have been reported within the last 48 hours as having run attacks attacks on the RFI-Attacks, REG-Bots, IRC-Bots or BadBots (BadBots = he has posted a Spam-Comment on a open Forum or Wiki). - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/bots.txt - name: Bots - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### Brute-force Logins - -Blocklist.DE Brute-force Login Collector is the bot responsible to get the report from source of information. All IPs which attacks Joomlas, Wordpress and other Web-Logins with Brute-Force Logins. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/bruteforcelogin.txt - name: Brute-force Logins - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### FTP - -Blocklist.DE FTP Collector is the bot responsible to get the report from source of information. All IP addresses which have been reported within the last 48 hours for attacks on the Service FTP. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/ftp.txt - name: FTP - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### IMAP - -Blocklist.DE IMAP Collector is the bot responsible to get the report from source of information. All IP addresses which have been reported within the last 48 hours for attacks on the service like IMAP, SASL, POP3, etc. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/imap.txt - name: IMAP - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### IRC Bots - -No description provided by feed provider. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/ircbot.txt - name: IRC Bots - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### Mail - -Blocklist.DE Mail Collector is the bot responsible to get the report from source of information. All IP addresses which have been reported within the last 48 hours as having run attacks on the service Mail, Postfix. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/mail.txt - name: Mail - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### SIP - -Blocklist.DE SIP Collector is the bot responsible to get the report from source of information. All IP addresses that tried to login in a SIP-, VOIP- or Asterisk-Server and are included in the IPs-List from http://www.infiltrated.net/ (Twitter). - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/sip.txt - name: SIP - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### SSH - -Blocklist.DE SSH Collector is the bot responsible to get the report from source of information. All IP addresses which have been reported within the last 48 hours as having run attacks on the service SSH. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/ssh.txt - name: SSH - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -### Strong IPs - -Blocklist.DE Strong IPs Collector is the bot responsible to get the report from source of information. All IPs which are older then 2 month and have more then 5.000 attacks. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.blocklist.de/lists/strongips.txt - name: Strong IPs - provider: Blocklist.de - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blocklistde.parser -``` - ---- - - -## Blueliv - -### CrimeServer - -Blueliv Crimeserver Collector is the bot responsible to get the report through the API. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - -**Additional Information:** The service uses a different API for free users and paying subscribers. In 'CrimeServer' feed the difference lies in the data points present in the feed. The non-free API available from Blueliv contains, for this specific feed, following extra fields not present in the free API; "_id" - Internal unique ID "subType" - Subtype of the Crime Server "countryName" - Country name where the Crime Server is located, in English "city" - City where the Crime Server is located "domain" - Domain of the Crime Server "host" - Host of the Crime Server "createdAt" - Date when the Crime Server was added to Blueliv CrimeServer database "asnCidr" - Range of IPs that belong to an ISP (registered via Autonomous System Number (ASN)) "asnId" - Identifier of an ISP registered via ASN "asnDesc" Description of the ISP registered via ASN - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.blueliv.collector_crimeserver -parameters: - api_key: __APIKEY__ - name: CrimeServer - provider: Blueliv - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.blueliv.parser_crimeserver -``` - ---- - - -## CERT-Bund - -### CB-Report Malware infections via IMAP - -CERT-Bund sends reports for the malware-infected hosts. - -**Public:** no - -**Revision:** 2020-08-20 - -**Additional Information:** Traffic from malware related hosts contacting command-and-control servers is caught and sent to national CERT teams. There are two e-mail feeds with identical CSV structure -- one reports on general malware infections, the other on the Avalanche botnet. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.mail.collector_mail_attach -parameters: - attach_regex: events.csv - extract_files: False - folder: INBOX - mail_host: __HOST__ - mail_password: __PASSWORD__ - mail_ssl: True - mail_user: __USERNAME__ - name: CB-Report Malware infections via IMAP - provider: CERT-Bund - rate_limit: 86400 - subject_regex: ^\\[CB-Report#.* Malware infections (\\(Avalanche\\) )?in country -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.generic.parser_csv -parameters: - columns: ["source.asn", "source.ip", "time.source", "classification.type", "malware.name", "source.port", "destination.ip", "destination.port", "destination.fqdn", "protocol.transport"] - default_url_protocol: http:// - defaults_fields: {'classification.type': 'infected-system'} - delimiter: , - skip_header: True - time_format: from_format|%Y-%m-%d %H:%M:%S -``` - ---- - - -## CERT.PL - -### N6 Stomp Stream - -N6 Collector - CERT.pl's *n6* Stream API feed (via STOMP interface). Note that 'rate_limit' does not apply to this bot, as it is waiting for messages on a stream. - -**Public:** no - -**Revision:** 2023-10-08 - -**Documentation:** - -**Additional Information:** Contact CERT.pl to get access to the feed. Note that the configuration parameter values suggested here are suitable for the new *n6* Stream API variant (with authentication based on 'username' and 'password'); for this variant, typically you can leave the 'ssl_ca_certificate' parameter's value empty - then the system's default CA certificates will be used; however, if that does not work, you need to set 'ssl_ca_certificate' to the path to a file containing CA certificates eligible to verify "*.cert.pl" server certificates (to be found among the publicly available CA certs distributed with modern web browsers/OSes). Also, note that the 'server' parameter's value (for the *new API variant*) suggested here, "n6stream-new.cert.pl", is a temporary domain; ultimately, it will be changed back to "stream.cert.pl". When it comes to the *old API variant* (turned off in November 2023!), you need to have the 'server' parameter set to the name "n6stream.cert.pl", 'auth_by_ssl_client_certificate' set to true, 'ssl_ca_certificate' set to the path to a file containing the *n6*'s legacy self-signed CA certificate (which is stored in file "intelmq/bots/collectors/stomp/ca.pem"), and the parameters 'ssl_client_certificate' and 'ssl_client_certificate_key' set to the paths to your-*n6*-client-specific certificate and key files (note that the 'username' and 'password' parameters are then irrelevant and can be omitted). - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.stomp.collector -parameters: - auth_by_ssl_client_certificate: False - exchange: {insert your STOMP *destination* to subscribe to, as given by CERT.pl, e.g. /exchange/my.example.org/*.*.*.*} - name: N6 Stomp Stream - password: {insert your *n6* API key} - port: 61614 - provider: CERT.PL - server: n6stream-new.cert.pl - ssl_ca_certificate: - username: {insert your *n6* login, e.g. someuser@my.example.org} -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.n6.parser_n6stomp -``` - ---- - - -## CINS Army - -### CINS Army List - -The CINS Army (CIArmy.com) list is a subset of the CINS Active Threat Intelligence ruleset, and consists of IP addresses that meet one of two basic criteria: 1) The IP's recent Rogue Packet score factor is very poor, or 2) The IP has tripped a designated number of 'trusted' alerts across a given number of our Sentinels deployed around the world. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://cinsscore.com/list/ci-badguys.txt - name: CINS Army List - provider: CINS Army - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.ci_army.parser -``` - ---- - - -## CZ.NIC - -### HaaS - -SSH attackers against HaaS (Honeypot as a Service) provided by CZ.NIC, z.s.p.o. The dump is published once a day. - -**Public:** yes - -**Revision:** 2020-07-22 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - extract_files: True - http_url: https://haas.nic.cz/stats/export/{time[%Y/%m/%Y-%m-%d]}.json.gz - http_url_formatting: {'days': -1} - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.cznic.parser_haas -``` - ---- - - -### Proki - -Aggregation of various sources on malicious IP addresses (malware spreaders or C&C servers). - -**Public:** no - -**Revision:** 2020-08-17 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://proki.csirt.cz/api/1/__APIKEY__/data/day/{time[%Y/%m/%d]} - http_url_formatting: {'days': -1} - name: Proki - provider: CZ.NIC - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.cznic.parser_proki -``` - ---- - - -## Calidog - -### CertStream - -HTTP Websocket Stream from certstream.calidog.io providing data from Certificate Transparency Logs. - -**Public:** yes - -**Revision:** 2018-06-15 - -**Documentation:** - -**Additional Information:** Be aware that this feed provides a lot of data and may overload your system quickly. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.calidog.collector_certstream -parameters: - name: CertStream - provider: Calidog -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.calidog.parser_certstream -``` - ---- - - -## CleanMX - -### Phishing - -In order to download the CleanMX feed you need to use a custom user agent and register that user agent. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_timeout_sec: 120 - http_url: http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&domain= - http_user_agent: {{ your user agent }} - name: Phishing - provider: CleanMX - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.cleanmx.parser -``` - ---- - - -### Virus - -In order to download the CleanMX feed you need to use a custom user agent and register that user agent. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_timeout_sec: 120 - http_url: http://support.clean-mx.de/clean-mx/xmlviruses?response=alive&domain= - http_user_agent: {{ your user agent }} - name: Virus - provider: CleanMX - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.cleanmx.parser -``` - ---- - - -## CyberCrime Tracker - -### Latest - -C2 servers - -**Public:** yes - -**Revision:** 2019-03-19 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://cybercrime-tracker.net/index.php - name: Latest - provider: CyberCrime Tracker - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.html_table.parser -parameters: - columns: ["time.source", "source.url", "source.ip", "malware.name", "__IGNORE__"] - default_url_protocol: http:// - defaults_fields: {'classification.type': 'c2-server'} - skip_table_head: True -``` - ---- - - -## DShield - -### AS Details - -No description provided by feed provider. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dshield.org/asdetailsascii.html?as={{ AS Number }} - name: AS Details - provider: DShield - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dshield.parser_asn -``` - ---- - - -### Block - -This list summarizes the top 20 attacking class C (/24) subnets over the last three days. The number of 'attacks' indicates the number of targets reporting scans from this subnet. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.dshield.org/block.txt - name: Block - provider: DShield - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dshield.parser_block -``` - ---- - - -## Danger Rulez - -### Bruteforce Blocker - -Its main purpose is to block SSH bruteforce attacks via firewall. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://danger.rulez.sk/projects/bruteforceblocker/blist.php - name: Bruteforce Blocker - provider: Danger Rulez - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.danger_rulez.parser -``` - ---- - - -## Dataplane - -### DNS Recursion Desired - -Entries consist of fields with identifying characteristics of a source IP address that has been seen performing a DNS recursion desired query to a remote host. This report lists hosts that are suspicious of more than just port scanning. The host may be DNS server cataloging or searching for hosts to use for DNS-based DDoS amplification. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/dnsrd.txt - name: DNS Recursion Desired - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### DNS Recursion Desired ANY - -Entries consist of fields with identifying characteristics of a source IP address that has been seen performing a DNS recursion desired IN ANY query to a remote host. This report lists hosts that are suspicious of more than just port scanning. The host may be DNS server cataloging or searching for hosts to use for DNS-based DDoS amplification. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/dnsrdany.txt - name: DNS Recursion Desired ANY - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### DNS Version - -Entries consist of fields with identifying characteristics of a source IP address that has been seen performing a DNS CH TXT version.bind query to a remote host. This report lists hosts that are suspicious of more than just port scanning. The host may be DNS server cataloging or searching for vulnerable DNS servers. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/dnsversion.txt - name: DNS Version - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### Protocol 41 - -Entries consist of fields with identifying characteristics of a host that has been detected to offer open IPv6 over IPv4 tunneling. This could allow for the host to be used a public proxy against IPv6 hosts. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/proto41.txt - name: Protocol 41 - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### SIP Query - -Entries consist of fields with identifying characteristics of a source IP address that has been seen initiating a SIP OPTIONS query to a remote host. This report lists hosts that are suspicious of more than just port scanning. The hosts may be SIP server cataloging or conducting various forms of telephony abuse. Report is updated hourly. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/sipquery.txt - name: SIP Query - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### SIP Registration - -Entries consist of fields with identifying characteristics of a source IP address that has been seen initiating a SIP REGISTER operation to a remote host. This report lists hosts that are suspicious of more than just port scanning. The hosts may be SIP client cataloging or conducting various forms of telephony abuse. Report is updated hourly. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/sipregistration.txt - name: SIP Registration - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### SMTP Data - -Entries consist of fields with identifying characteristics of a host that has been seen initiating a SMTP DATA operation to a remote host. The source report lists hosts that are suspicious of more than just port scanning. The host may be SMTP server cataloging or conducting various forms of email abuse. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/smtpdata.txt - name: SMTP Data - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### SMTP Greet - -Entries consist of fields with identifying characteristics of a host that has been seen initiating a SMTP HELO/EHLO operation to a remote host. The source report lists hosts that are suspicious of more than just port scanning. The host may be SMTP server cataloging or conducting various forms of email abuse. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/smtpgreet.txt - name: SMTP Greet - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### SSH Client Connection - -Entries below consist of fields with identifying characteristics of a source IP address that has been seen initiating an SSH connection to a remote host. This report lists hosts that are suspicious of more than just port scanning. The hosts may be SSH server cataloging or conducting authentication attack attempts. Report is updated hourly. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/sshclient.txt - name: SSH Client Connection - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### SSH Password Authentication - -Entries below consist of fields with identifying characteristics of a source IP address that has been seen attempting to remotely login to a host using SSH password authentication. The report lists hosts that are highly suspicious and are likely conducting malicious SSH password authentication attacks. Report is updated hourly. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/sshpwauth.txt - name: SSH Password Authentication - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### Telnet Login - -Entries consist of fields with identifying characteristics of a host that has been seen initiating a telnet connection to a remote host. The source report lists hosts that are suspicious of more than just port scanning. The host may be telnet server cataloging or conducting authentication attack attempts. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/telnetlogin.txt - name: Telnet Login - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -### VNC/RFB Login - -Entries consist of fields with identifying characteristics of a host that has been seen initiating a VNC remote buffer session to a remote host. The source report lists hosts that are suspicious of more than just port scanning. The host may be VNC/RFB server cataloging or conducting authentication attack attempts. - -**Public:** yes - -**Revision:** 2021-09-09 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://dataplane.org/vncrfb.txt - name: VNC/RFB Login - provider: Dataplane - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dataplane.parser -``` - ---- - - -## ESET - -### ETI Domains - -Domain data from ESET's TAXII API. - -**Public:** no - -**Revision:** 2020-06-30 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.eset.collector -parameters: - collection: ei.domains v2 (json) - endpoint: eti.eset.com - password: - time_delta: 3600 - username: -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.eset.parser -``` - ---- - - -### ETI URLs - -URL data from ESET's TAXII API. - -**Public:** no - -**Revision:** 2020-06-30 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.eset.collector -parameters: - collection: ei.urls (json) - endpoint: eti.eset.com - password: - time_delta: 3600 - username: -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.eset.parser -``` - ---- - - -## Fireeye - -### Malware Analysis System - -Process data from Fireeye mail and file analysis appliances. SHA1 and MD5 malware hashes are extracted and if there is network communication, also URLs and domains. - -**Public:** no - -**Revision:** 2021-05-03 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.fireeye.collector_mas -parameters: - host: - http_password: - http_username: - request_duration: -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.fireeye.parser -``` - ---- - - -## Fraunhofer - -### DGA Archive - -Fraunhofer DGA collector fetches data from Fraunhofer's domain generation archive. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_password: {{ your password }} - http_url: https://dgarchive.caad.fkie.fraunhofer.de/today - http_username: {{ your username }} - name: DGA Archive - provider: Fraunhofer - rate_limit: 10800 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.fraunhofer.parser_dga -``` - ---- - - -## Have I Been Pwned - -### Enterprise Callback - -With the Enterprise Subscription of 'Have I Been Pwned' you are able to provide a callback URL and any new leak data is submitted to it. It is recommended to put a webserver with Authorization check, TLS etc. in front of the API collector. - -**Public:** no - -**Revision:** 2019-09-11 - -**Documentation:** - -**Additional Information:** A minimal nginx configuration could look like: -``` -server { - listen 443 ssl http2; - server_name [your host name]; - client_max_body_size 50M; - - ssl_certificate [path to your key]; - ssl_certificate_key [path to your certificate]; - - location /[your private url] { - if ($http_authorization != '[your private password]') { - return 403; - } - proxy_pass http://localhost:5001/intelmq/push; - proxy_read_timeout 30; - proxy_connect_timeout 30; - } -} -``` - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.api.collector_api -parameters: - name: Enterprise Callback - port: 5001 - provider: Have I Been Pwned -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.hibp.parser_callback -``` - ---- - - -## MalwarePatrol - -### DansGuardian - -Malware block list with URLs - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://lists.malwarepatrol.net/cgi/getfile?receipt={{ your API key }}&product=8&list=dansguardian - name: DansGuardian - provider: MalwarePatrol - rate_limit: 180000 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.malwarepatrol.parser_dansguardian -``` - ---- - - -## MalwareURL - -### Latest malicious activity - -Latest malicious domains/IPs. - -**Public:** yes - -**Revision:** 2018-02-05 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.malwareurl.com/ - name: Latest malicious activity - provider: MalwareURL - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.malwareurl.parser -``` - ---- - - -## McAfee Advanced Threat Defense - -### Sandbox Reports - -Processes reports from McAfee's sandboxing solution via the openDXL API. - -**Public:** no - -**Revision:** 2018-07-05 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.opendxl.collector -parameters: - dxl_config_file: {{ location of dxl configuration file }} - dxl_topic: /mcafee/event/atd/file/report -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.mcafee.parser_atd -parameters: - verdict_severity: 4 -``` - ---- - - -## Microsoft - -### BingMURLs via Interflow - -Collects Malicious URLs detected by Bing from the Interflow API. The feed is available via Microsoft’s Government Security Program (GSP). - -**Public:** no - -**Revision:** 2018-05-29 - -**Documentation:** - -**Additional Information:** Depending on the file sizes you may need to increase the parameter 'http_timeout_sec' of the collector. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.microsoft.collector_interflow -parameters: - api_key: {{ your API key }} - file_match: ^bingmurls_ - http_timeout_sec: 300 - name: BingMURLs via Interflow - not_older_than: 2 days - provider: Microsoft - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.microsoft.parser_bingmurls -``` - ---- - - -### CTIP C2 via Azure - -Collects the CTIP C2 feed from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP). - -**Public:** no - -**Revision:** 2020-05-29 - -**Documentation:** - -**Additional Information:** The cache is needed for memorizing which files have already been processed, the TTL should be higher than the oldest file available in the storage (currently the last three days are available). The connection string contains endpoint as well as authentication information. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.microsoft.collector_azure -parameters: - connection_string: {{ your connection string }} - container_name: ctip-c2 - name: CTIP C2 via Azure - provider: Microsoft - rate_limit: 3600 - redis_cache_db: 5 - redis_cache_host: 127.0.0.1 - redis_cache_port: 6379 - redis_cache_ttl: 864000 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.microsoft.parser_ctip -``` - ---- - - -### CTIP Infected via Azure - -Collects the CTIP (Sinkhole data) from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP). - -**Public:** no - -**Revision:** 2022-06-01 - -**Documentation:** - -**Additional Information:** The cache is needed for memorizing which files have already been processed, the TTL should be higher than the oldest file available in the storage (currently the last three days are available). The connection string contains endpoint as well as authentication information. As many IPs occur very often in the data, you may want to use a deduplicator specifically for the feed. More information about the feed can be found on www.dcuctip.com after login with your GSP account. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.microsoft.collector_azure -parameters: - connection_string: {{ your connection string }} - container_name: ctip-infected-summary - name: CTIP Infected via Azure - provider: Microsoft - rate_limit: 3600 - redis_cache_db: 5 - redis_cache_host: 127.0.0.1 - redis_cache_port: 6379 - redis_cache_ttl: 864000 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.microsoft.parser_ctip -``` - ---- - - -### CTIP Infected via Interflow - -Collects the CTIP Infected feed (Sinkhole data for your country) files from the Interflow API.The feed is available via Microsoft’s Government Security Program (GSP). - -**Public:** no - -**Revision:** 2018-03-06 - -**Documentation:** - -**Additional Information:** Depending on the file sizes you may need to increase the parameter 'http_timeout_sec' of the collector. As many IPs occur very often in the data, you may want to use a deduplicator specifically for the feed. More information about the feed can be found on www.dcuctip.com after login with your GSP account. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.microsoft.collector_interflow -parameters: - api_key: {{ your API key }} - file_match: ^ctip_ - http_timeout_sec: 300 - name: CTIP Infected via Interflow - not_older_than: 2 days - provider: Microsoft - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.microsoft.parser_ctip -``` - ---- - - -## OpenPhish - -### Premium Feed - -OpenPhish is a fully automated self-contained platform for phishing intelligence. It identifies phishing sites and performs intelligence analysis in real time without human intervention and without using any external resources, such as blacklists. - -**Public:** no - -**Revision:** 2018-02-06 - -**Documentation:** - -**Additional Information:** Discounts available for Government and National CERTs a well as for Nonprofit and Not-for-Profit organizations. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_password: {{ your password }} - http_url: https://openphish.com/prvt-intell/ - http_username: {{ your username }} - name: Premium Feed - provider: OpenPhish - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.openphish.parser_commercial -``` - ---- - - -### Public feed - -OpenPhish is a fully automated self-contained platform for phishing intelligence. It identifies phishing sites and performs intelligence analysis in real time without human intervention and without using any external resources, such as blacklists. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.openphish.com/feed.txt - name: Public feed - provider: OpenPhish - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.openphish.parser -``` - ---- - - -## PhishTank - -### Online - -PhishTank is a collaborative clearing house for data and information about phishing on the Internet. - -**Public:** no - -**Revision:** 2022-11-21 - -**Documentation:** - -**Additional Information:** Updated hourly as per the documentation. Download is possible without API key, but limited to few downloads per day. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - extract_files: True - http_url: https://data.phishtank.com/data/{{ your API key }}/online-valid.json.gz - name: Online - provider: PhishTank - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.phishtank.parser -``` - ---- - - -## PrecisionSec - -### Agent Tesla - -Agent Tesla IoCs, URLs where the malware is hosted. - -**Public:** yes - -**Revision:** 2019-04-02 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://precisionsec.com/threat-intelligence-feeds/agent-tesla/ - name: Agent Tesla - provider: PrecisionSec - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.html_table.parser -parameters: - columns: ["source.ip|source.url", "time.source"] - default_url_protocol: http:// - defaults_fields: {'classification.type': 'malware-distribution'} - skip_table_head: True -``` - ---- - - -## Shadowserver - -### Via API - -Shadowserver sends out a variety of reports to subscribers, see documentation. - -**Public:** no - -**Revision:** 2020-01-08 - -**Documentation:** - -**Additional Information:** This configuration fetches user-configurable reports from the Shadowserver Reports API. For a list of reports, have a look at the Shadowserver collector and parser documentation. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.shadowserver.collector_reports_api -parameters: - api_key: - country: - rate_limit: 86400 - redis_cache_db: 12 - redis_cache_host: 127.0.0.1 - redis_cache_port: 6379 - redis_cache_ttl: 864000 - secret: - types: -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.shadowserver.parser_json -``` - ---- - - -### Via IMAP - -Shadowserver sends out a variety of reports (see https://www.shadowserver.org/wiki/pmwiki.php/Services/Reports). - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - -**Additional Information:** The configuration retrieves the data from a e-mails via IMAP from the attachments. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.mail.collector_mail_attach -parameters: - attach_regex: csv.zip - extract_files: True - folder: INBOX - mail_host: __HOST__ - mail_password: __PASSWORD__ - mail_ssl: True - mail_user: __USERNAME__ - name: Via IMAP - provider: Shadowserver - rate_limit: 86400 - subject_regex: __REGEX__ -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.shadowserver.parser -``` - ---- - - -### Via Request Tracker - -Shadowserver sends out a variety of reports (see https://www.shadowserver.org/wiki/pmwiki.php/Services/Reports). - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - -**Additional Information:** The configuration retrieves the data from a RT/RTIR ticketing instance via the attachment or an download. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.rt.collector_rt -parameters: - attachment_regex: \\.csv\\.zip$ - extract_attachment: True - extract_download: False - http_password: {{ your HTTP Authentication password or null }} - http_username: {{ your HTTP Authentication username or null }} - password: __PASSWORD__ - provider: Shadowserver - rate_limit: 3600 - search_not_older_than: {{ relative time or null }} - search_owner: nobody - search_queue: Incident Reports - search_requestor: autoreports@shadowserver.org - search_status: new - search_subject_like: \[__COUNTRY__\] Shadowserver __COUNTRY__ - set_status: open - take_ticket: True - uri: http://localhost/rt/REST/1.0 - url_regex: https://dl.shadowserver.org/[a-zA-Z0-9?_-]* - user: __USERNAME__ -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.shadowserver.parser -``` - ---- - - -## Shodan - -### Country Stream - -Collects the Shodan stream for one or multiple countries from the Shodan API. - -**Public:** no - -**Revision:** 2021-03-22 - -**Documentation:** - -**Additional Information:** A Shodan account with streaming permissions is needed. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.shodan.collector_stream -parameters: - api_key: - countries: - error_retry_delay: 0 - name: Country Stream - provider: Shodan -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.shodan.parser -parameters: - error_retry_delay: 0 - ignore_errors: False - minimal_mode: False -``` - ---- - - -## Spamhaus - -### ASN Drop - -ASN-DROP contains a list of Autonomous System Numbers controlled by spammers or cyber criminals, as well as "hijacked" ASNs. ASN-DROP can be used to filter BGP routes which are being used for malicious purposes. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.spamhaus.org/drop/asndrop.txt - name: ASN Drop - provider: Spamhaus - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.spamhaus.parser_drop -``` - ---- - - -### CERT - -Spamhaus CERT Insight Portal. Access limited to CERTs and CSIRTs with national or regional responsibility. . - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: {{ your CERT portal URL }} - name: CERT - provider: Spamhaus - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.spamhaus.parser_cert -``` - ---- - - -### Drop - -The DROP list will not include any IP address space under the control of any legitimate network - even if being used by "the spammers from hell". DROP will only include netblocks allocated directly by an established Regional Internet Registry (RIR) or National Internet Registry (NIR) such as ARIN, RIPE, AFRINIC, APNIC, LACNIC or KRNIC or direct RIR allocations. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.spamhaus.org/drop/drop.txt - name: Drop - provider: Spamhaus - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.spamhaus.parser_drop -``` - ---- - - -### Dropv6 - -The DROPv6 list includes IPv6 ranges allocated to spammers or cyber criminals. DROPv6 will only include IPv6 netblocks allocated directly by an established Regional Internet Registry (RIR) or National Internet Registry (NIR) such as ARIN, RIPE, AFRINIC, APNIC, LACNIC or KRNIC or direct RIR allocations. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.spamhaus.org/drop/dropv6.txt - name: Dropv6 - provider: Spamhaus - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.spamhaus.parser_drop -``` - ---- - - -### EDrop - -EDROP is an extension of the DROP list that includes sub-allocated netblocks controlled by spammers or cyber criminals. EDROP is meant to be used in addition to the direct allocations on the DROP list. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.spamhaus.org/drop/edrop.txt - name: EDrop - provider: Spamhaus - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.spamhaus.parser_drop -``` - ---- - - -## Strangereal Intel - -### DailyIOC - -Daily IOC from tweets and articles - -**Public:** yes - -**Revision:** 2019-12-05 - -**Documentation:** - -**Additional Information:** collector's `extra_fields` parameter may be any of fields from the github `content API response `_ - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.github_api.collector_github_contents_api -parameters: - personal_access_token: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token - regex: .*.json - repository: StrangerealIntel/DailyIOC -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.github_feed -``` - ---- - - -## Surbl - -### Malicious Domains - -Detected malicious domains. Note that you have to opened up Sponsored Datafeed Service (SDS) access to the SURBL data via rsync for your IP address. - -**Public:** no - -**Revision:** 2018-09-04 - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.rsync.collector_rsync -parameters: - file: wild.surbl.org.rbldnsd - rsync_path: blacksync.prolocation.net::surbl-wild/ -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.surbl.parser -``` - ---- - - -## Team Cymru - -### CAP - -Team Cymru provides daily lists of compromised or abused devices for the ASNs and/or netblocks with a CSIRT's jurisdiction. This includes such information as bot infected hosts, command and control systems, open resolvers, malware urls, phishing urls, and brute force attacks - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - -**Additional Information:** "Two feeds types are offered: - * The new https://www.cymru.com/$certname/$certname_{time[%Y%m%d]}.txt - * and the old https://www.cymru.com/$certname/infected_{time[%Y%m%d]}.txt - Both formats are supported by the parser and the new one is recommended. - As of 2019-09-12 the old format will be retired soon." - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_password: {{ your password }} - http_url: https://www.cymru.com/$certname/$certname_{time[%Y%m%d]}.txt - http_url_formatting: True - http_username: {{ your username }} - name: CAP - provider: Team Cymru - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.cymru.parser_cap_program -``` - ---- - - -### Full Bogons IPv4 - -Fullbogons are a larger set which also includes IP space that has been allocated to an RIR, but not assigned by that RIR to an actual ISP or other end-user. IANA maintains a convenient IPv4 summary page listing allocated and reserved netblocks, and each RIR maintains a list of all prefixes that they have assigned to end-users. Our bogon reference pages include additional links and resources to assist those who wish to properly filter bogon prefixes within their networks. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt - name: Full Bogons IPv4 - provider: Team Cymru - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.cymru.parser_full_bogons -``` - ---- - - -### Full Bogons IPv6 - -Fullbogons are a larger set which also includes IP space that has been allocated to an RIR, but not assigned by that RIR to an actual ISP or other end-user. IANA maintains a convenient IPv4 summary page listing allocated and reserved netblocks, and each RIR maintains a list of all prefixes that they have assigned to end-users. Our bogon reference pages include additional links and resources to assist those who wish to properly filter bogon prefixes within their networks. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt - name: Full Bogons IPv6 - provider: Team Cymru - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.cymru.parser_full_bogons -``` - ---- - - -## Threatminer - -### Recent domains - -Latest malicious domains. - -**Public:** yes - -**Revision:** 2018-02-06 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.threatminer.org/ - name: Recent domains - provider: Threatminer - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.threatminer.parser -``` - ---- - - -## Turris - -### Greylist - -The data are processed and classified every week and behaviour of IP addresses that accessed a larger number of Turris routers is evaluated. The result is a list of addresses that have tried to obtain information about services on the router or tried to gain access to them. The list also contains a list of tags for each address which indicate what behaviour of the address was observed. - -**Public:** yes - -**Revision:** 2023-06-13 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://view.sentinel.turris.cz/greylist-data/greylist-latest.csv - name: Greylist - provider: Turris - rate_limit: 43200 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.turris.parser -``` - ---- - - -### Greylist with PGP signature verification - -The data are processed and classified every week and behaviour of -IP addresses that accessed a larger number of Turris routers is evaluated. -The result is a list of addresses that have tried to obtain information about -services on the router or tried to gain access to them. The list also -contains a list of tags for each address which -indicate what behaviour of the address was observed. - -The Turris Greylist feed provides PGP signatures for the provided files. -You will need to import the public PGP key from the linked documentation -page, currently available at -https://pgp.mit.edu/pks/lookup?op=vindex&search=0x10876666 -or from below. -See the URL Fetcher Collector documentation for more information on -PGP signature verification. - -PGP Public key: - -``` ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: SKS 1.1.6 -Comment: Hostname: pgp.mit.edu - -mQINBFRl7D8BEADaRFoDa/+r27Gtqrdn8sZL4aSYTU4Q3gDr3TfigK8H26Un/Y79a/DUL1o0 -o8SRae3uwVcjJDHZ6KDnxThbqF7URfpuCcCYxOs8p/eu3dSueqEGTODHWF4ChIh2japJDc4t -3FQHbIh2e3GHotVqJGhvxMmWqBFoZ/mlWvhjs99FFBZ87qbUNk7l1UAGEXeWeECgz9nGox40 -3YpCgEsnJJsKC53y5LD/wBf4z+z0GsLg2GMRejmPRgrkSE/d9VjF/+niifAj2ZVFoINSVjjI -8wQFc8qLiExdzwLdgc+ggdzk5scY3ugI5IBt1zflxMIOG4BxKj/5IWsnhKMG2NLVGUYOODoG -pKhcY0gCHypw1bmkp2m+BDVyg4KM2fFPgQ554DAX3xdukMCzzZyBxR3UdT4dN7xRVhpph3Y2 -Amh1E/dpde9uwKFk1oRHkRZ3UT1XtpbXtFNY0wCiGXPt6KznJAJcomYFkeLHjJo3nMK0hISV -GSNetVLfNWlTkeo93E1innbSaDEN70H4jPivjdVjSrLtIGfr2IudUJI84dGmvMxssWuM2qdg -FSzoTHw9UE9KT3SltKPS+F7u9x3h1J492YaVDncATRjPZUBDhbvo6Pcezhup7XTnI3gbRQc2 -oEUDb933nwuobHm3VsUcf9686v6j8TYehsbjk+zdA4BoS/IdCwARAQABtC5UdXJyaXMgR3Jl -eWxpc3QgR2VuZXJhdG9yIDxncmV5bGlzdEB0dXJyaXMuY3o+iQI4BBMBAgAiBQJUZew/AhsD -BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRDAQrU3EIdmZoH4D/9Jo6j9RZxCAPTaQ9WZ -WOdb1Eqd/206bObEX+xJAago+8vuy+waatHYBM9/+yxh0SIg2g5whd6J7A++7ePpt5XzX6hq -bzdG8qGtsCRu+CpDJ40UwHep79Ck6O/A9KbZcZW1z/DhbYT3z/ZVWALy4RtgmyC67Vr+j/C7 -KNQ529bs3kP9AzvEIeBC4wdKl8dUSuZIPFbgf565zRNKLtHVgVhiuDPcxKmBEl4/PLYF30a9 -5Tgp8/PNa2qp1DV/EZjcsxvSRIZB3InGBvdKdSzvs4N/wLnKWedj1GGm7tJhSkJa4MLBSOIx -yamhTS/3A5Cd1qoDhLkp7DGVXSdgEtpoZDC0jR7nTS6pXojcgQaF7SfJ3cjZaLI5rjsx0YLk -G4PzonQKCAAQG1G9haCDniD8NrrkZ3eFiafoKEECRFETIG0BJHjPdSWcK9jtNCupBYb7JCiz -Q0hwLh2wrw/wCutQezD8XfsBFFIQC18TsJAVgdHLZnGYkd5dIbV/1scOcm52w6EGIeMBBYlB -J2+JNukH5sJDA6zAXNl2I1H1eZsP4+FSNIfB6LdovHVPAjn7qXCw3+IonnQK8+g8YJkbbhKJ -sPejfg+ndpe5u0zX+GvQCFBFu03muANA0Y/OOeGIQwU93d/akN0P1SRfq+bDXnkRIJQOD6XV -0ZPKVXlNOjy/z2iN2A== -=wjkM ------END PGP PUBLIC KEY BLOCK----- -``` - - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://www.turris.cz/greylist-data/greylist-latest.csv - name: Greylist - provider: Turris - rate_limit: 43200 - signature_url: https://www.turris.cz/greylist-data/greylist-latest.csv.asc - verify_pgp_signatures: True -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.turris.parser -``` - ---- - - -## University of Toulouse - -### Blacklist - -Various blacklist feeds - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - extract_files: true - http_url: https://dsi.ut-capitole.fr/blacklists/download/{collection name}.tar.gz - name: Blacklist - provider: University of Toulouse - rate_limit: 43200 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.generic.parser_csv -parameters: - columns: {depends on a collection} - defaults_fields: {'classification.type': '{depends on a collection}'} - delimiter: false -``` - ---- - - -## VXVault - -### URLs - -This feed provides IP addresses hosting Malware. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://vxvault.net/URL_List.php - name: URLs - provider: VXVault - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.vxvault.parser -``` - ---- - - -## ViriBack - -### C2 Tracker - -Latest detected C2 servers. - -**Public:** yes - -**Revision:** 2022-11-15 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://tracker.viriback.com/dump.php - name: C2 Tracker - provider: ViriBack - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.generic.csv_parser -parameters: - columns: ["malware.name", "source.url", "source.ip", "time.source"] - defaults_fields: {'classification.type': 'malware-distribution'} - skip_header: True -``` - ---- - - -## ZoneH - -### Defacements - -all the information contained in Zone-H's cybercrime archive were either collected online from public sources or directly notified anonymously to us. - -**Public:** no - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.mail.collector_mail_attach -parameters: - attach_regex: csv - extract_files: False - folder: INBOX - mail_host: __HOST__ - mail_password: __PASSWORD__ - mail_ssl: True - mail_user: __USERNAME__ - name: Defacements - provider: ZoneH - rate_limit: 3600 - sent_from: datazh@zone-h.org - subject_regex: Report -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.zoneh.parser -``` - ---- - - -## cAPTure - -### Ponmocup Domains CIF Format - -List of ponmocup malware redirection domains and infected web-servers from cAPTure. See also http://security-research.dyndns.org/pub/botnet-links.htm and http://c-apt-ure.blogspot.com/search/label/ponmocup The data in the CIF format is not equal to the Shadowserver CSV format. Reasons are unknown. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://security-research.dyndns.org/pub/malware-feeds/ponmocup-infected-domains-CIF-latest.txt - name: Infected Domains - provider: cAPTure - rate_limit: 10800 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.dyn.parser -``` - ---- - - -### Ponmocup Domains Shadowserver Format - -List of ponmocup malware redirection domains and infected web-servers from cAPTure. See also http://security-research.dyndns.org/pub/botnet-links.htm and http://c-apt-ure.blogspot.com/search/label/ponmocup The data in the Shadowserver CSV is not equal to the CIF format format. Reasons are unknown. - -**Public:** yes - -**Revision:** 2020-07-08 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://security-research.dyndns.org/pub/malware-feeds/ponmocup-infected-domains-shadowserver.csv - name: Infected Domains - provider: cAPTure - rate_limit: 10800 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.generic.parser_csv -parameters: - columns: ["time.source", "source.ip", "source.fqdn", "source.urlpath", "source.port", "protocol.application", "extra.tag", "extra.redirect_target", "extra.category"] - compose_fields: {'source.url': 'http://{0}{1}'} - defaults_fields: {'classification.type': 'malware-distribution'} - delimiter: , - skip_header: True -``` - ---- - - From 99c029f242cc7723f8870c9d3f0be515b8ddfbea Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 27 Jun 2024 18:07:27 +0200 Subject: [PATCH 2/2] documentation workflow: build generated doc parts generate event and feed documentation before building the documentation --- .github/workflows/build-docs.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index e0cc45f54..d0bfb9ba5 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -37,7 +37,12 @@ jobs: - name: "Install build dependencies" run: | - pip install mkdocs-material mike lunr pygments mkdocstrings[python] mkdocs-material mkdocs-glightbox mkdocs-redirects mkdocs-minify-plugin + pip install mkdocs-material mike lunr pygments mkdocstrings[python] mkdocs-material mkdocs-glightbox mkdocs-redirects mkdocs-minify-plugin ruamel.yaml + + - name: "Generate Feed and Event documentation" + run: | + python3 scripts/generate-feeds-docs.py + python3 scripts/generate-event-docs.py - name: "Prepare git" run: |