Skip to content

Commit 5b29315

Browse files
committed
fixed Thug and CI complaints + changelog + bump
1 parent c28aafa commit 5b29315

File tree

9 files changed

+100
-87
lines changed

9 files changed

+100
-87
lines changed

.github/CHANGELOG.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
[**Upgrade Guide**](https://intelowlproject.github.io/docs/IntelOwl/installation/#update-to-the-most-recent-version)
44

5+
## [v6.1.0](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.1.0)
6+
This release merges all the developments performed by our Google Summer of Code contributors for this year. The program has just ended. You can read the related blogs for more info about:
7+
- [Nilay Gupta](https://x.com/guptanilay1): [New analyzers for IntelOwl](https://intelowlproject.github.io/blogs/gsoc24_new_analyzers_for_intelowl)
8+
- [Aryan Bhokare](https://www.linkedin.com/in/aryan-b-3803751a7/): [New Documentation Site for IntelOwl and friends](https://intelowlproject.github.io/blogs/gsoc24_New_documentation_site_summary)
9+
10+
You'll get really tons of new analyzers this time to try out!
11+
12+
Plus we have a new official [documentation site](https://intelowlproject.github.io/docs/)! Please refer to this one from now onwards.
13+
514
## [v6.0.4](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.0.4)
615
Mostly adjusts and fixes with few new analyzers: Vulners and AILTypoSquatting Library.
716

.github/release_template.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
- [ ] (optional) If we changed/added Docker Analyzers, we need to configure Docker Hub / Dependabot properly.
44
- [ ] Update `CHANGELOG.md` for the new version
5-
- [ ] Change version number in `docs/source/schema.yml` and `docker/.env`
5+
- [ ] Change version number `docker/.env`
66
- [ ] Verify CI Tests
77
- [ ] Create release for the branch `develop`.
88
Write the following statement there (change the version number):

api_app/analyzers_manager/file_analyzers/pe_info.py

Lines changed: 52 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
11
# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
22
# See the file 'LICENSE' for copying permission.
33

4-
# this analyzer leverage a forked version of PEfile ...
5-
# ... that fixes one common problem encountered in a lot of analysis
6-
# original repository: https://github.com/erocarrera/pefile
7-
# forked repository: https://github.com/mlodic/pefile
8-
94
import logging
105
import os
116
from datetime import datetime
@@ -59,6 +54,55 @@ def dotnetpe(self):
5954
results["is_dotnet"] = False
6055
return results
6156

57+
@staticmethod
58+
def _extract_sections(pe):
59+
sections = []
60+
for section in pe.sections:
61+
try:
62+
name = section.Name.decode()
63+
except UnicodeDecodeError as e:
64+
name = "UnableToDecode"
65+
logger.warning(f"Unable to decode section {section.Name} exception {e}")
66+
section_item = {
67+
"name": name,
68+
"address": hex(section.VirtualAddress),
69+
"virtual_size": hex(section.Misc_VirtualSize),
70+
"size": section.SizeOfRawData,
71+
"entropy": section.get_entropy(),
72+
}
73+
sections.append(section_item)
74+
75+
return sections
76+
77+
@staticmethod
78+
def _extract_import_table(pe):
79+
import_table = []
80+
directory_entry_import = getattr(pe, "DIRECTORY_ENTRY_IMPORT", [])
81+
for entry in directory_entry_import:
82+
imp = {
83+
"entryname": entry.dll.decode() if entry.dll else None,
84+
"symbols": [],
85+
}
86+
for symbol in entry.imports:
87+
if symbol.name:
88+
imp["symbols"].append(symbol.name.decode())
89+
import_table.append(imp)
90+
return import_table
91+
92+
@staticmethod
93+
def _extract_export_table(full_dump):
94+
export_table = []
95+
for entry in full_dump.get("Exported symbols", []):
96+
symbol_name = entry.get("Name", None)
97+
# in case it is a dictionary, we do not mind it
98+
try:
99+
export_table.append(symbol_name.decode())
100+
except (UnicodeDecodeError, AttributeError) as e:
101+
logger.debug(f"PE info error while decoding export table symbols: {e}")
102+
# this is to reduce the output
103+
export_table = export_table[:100]
104+
return export_table
105+
62106
def run(self):
63107
results = {}
64108
results["dotnet"] = self.dotnetpe()
@@ -80,25 +124,7 @@ def run(self):
80124
elif pe.is_exe():
81125
results["type"] = "EXE"
82126

83-
sections = []
84-
for section in pe.sections:
85-
try:
86-
name = section.Name.decode()
87-
except UnicodeDecodeError as e:
88-
name = "UnableToDecode"
89-
logger.warning(
90-
f"Unable to decode section {section.Name} exception {e}"
91-
)
92-
section_item = {
93-
"name": name,
94-
"address": hex(section.VirtualAddress),
95-
"virtual_size": hex(section.Misc_VirtualSize),
96-
"size": section.SizeOfRawData,
97-
"entropy": section.get_entropy(),
98-
}
99-
sections.append(section_item)
100-
101-
results["sections"] = sections
127+
results["sections"] = self._extract_sections(pe)
102128

103129
machine_value = pe.FILE_HEADER.Machine
104130
results["machine"] = machine_value
@@ -127,32 +153,8 @@ def run(self):
127153
timestamp
128154
).strftime("%Y-%m-%d %H:%M:%S")
129155

130-
import_table = []
131-
directory_entry_import = getattr(pe, "DIRECTORY_ENTRY_IMPORT", [])
132-
for entry in directory_entry_import:
133-
imp = {
134-
"entryname": entry.dll.decode() if entry.dll else None,
135-
"symbols": [],
136-
}
137-
for symbol in entry.imports:
138-
if symbol.name:
139-
imp["symbols"].append(symbol.name.decode())
140-
import_table.append(imp)
141-
results["import_table"] = import_table
142-
143-
export_table = []
144-
for entry in full_dump.get("Exported symbols", []):
145-
symbol_name = entry.get("Name", None)
146-
# in case it is a dictionary, we do not mind it
147-
try:
148-
export_table.append(symbol_name.decode())
149-
except (UnicodeDecodeError, AttributeError) as e:
150-
logger.debug(
151-
f"PE info error while decoding export table symbols: {e}"
152-
)
153-
# this is to reduce the output
154-
export_table = export_table[:100]
155-
results["export_table"] = export_table
156+
results["import_table"] = self._extract_import_table(pe)
157+
results["export_table"] = self._extract_export_table(full_dump)
156158

157159
results["flags"] = full_dump.get("Flags", [])
158160

api_app/analyzers_manager/file_analyzers/thug_file.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,16 +47,17 @@ def run(self):
4747
# construct a valid dir name into which thug will save the result
4848
fname = str(self.filename).replace("/", "_").replace(" ", "_")
4949
tmp_dir = f"{fname}_{secrets.token_hex(4)}"
50+
tmp_dir_full_path = "/opt/deploy/thug" + tmp_dir
5051
# get the file to send
5152
binary = self.read_file_bytes()
5253
# append final arguments,
5354
# -n -> output directory
5455
# -l -> the local file to analyze
55-
args.extend(["-n", "/home/thug/" + tmp_dir, "-l", f"@{fname}"])
56+
args.extend(["-n", tmp_dir_full_path, "-l", f"@{fname}"])
5657
# make request parameters
5758
req_data = {
5859
"args": args,
59-
"callback_context": {"read_result_from": tmp_dir},
60+
"callback_context": {"read_result_from": tmp_dir_full_path},
6061
}
6162
req_files = {fname: binary}
6263

api_app/analyzers_manager/observable_analyzers/thug_url.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,13 @@ def run(self):
4646
args = self._thug_args_builder()
4747
# construct a valid directory name into which thug will save the result
4848
tmp_dir = secrets.token_hex(4)
49+
tmp_dir_full_path = "/opt/deploy/thug" + tmp_dir
4950
# make request data
50-
args.extend(["-n", "/home/thug/" + tmp_dir, self.observable_name])
51+
args.extend(["-n", tmp_dir_full_path, self.observable_name])
5152

5253
req_data = {
5354
"args": args,
54-
"callback_context": {"read_result_from": tmp_dir},
55+
"callback_context": {"read_result_from": tmp_dir_full_path},
5556
}
5657

5758
return self._docker_run(req_data=req_data, req_files=None)

api_app/serializers/plugin.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ class Meta:
3535
)
3636

3737
class CustomValueField(rfs.JSONField):
38-
def to_internal_value(self, data):
38+
@staticmethod
39+
def to_internal_value(data):
3940
if not data:
4041
raise ValidationError({"detail": "Empty insertion"})
4142
logger.info(f"verifying that value {data} ({type(data)}) is JSON compliant")
@@ -84,7 +85,8 @@ def to_representation(self, value):
8485
plugin_name = rfs.CharField()
8586
value = CustomValueField()
8687

87-
def validate_value_type(self, value: Any, parameter: Parameter):
88+
@staticmethod
89+
def validate_value_type(value: Any, parameter: Parameter):
8890
if type(value).__name__ != parameter.type:
8991
raise ValidationError(
9092
{
@@ -155,7 +157,8 @@ class Meta:
155157
fields = ["name", "type", "description", "required", "value", "is_secret"]
156158
list_serializer_class = ParamListSerializer
157159

158-
def get_value(self, param: Parameter):
160+
@staticmethod
161+
def get_value(param: Parameter):
159162
if hasattr(param, "value") and hasattr(param, "is_from_org"):
160163
if param.is_secret and param.is_from_org:
161164
return "redacted"

docker/.env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
### DO NOT CHANGE THIS VALUE !!
22
### It should be updated only when you pull latest changes off from the 'master' branch of IntelOwl.
33
# this variable must start with "REACT_APP_" to be used in the frontend too
4-
REACT_APP_INTELOWL_VERSION=v6.0.4
4+
REACT_APP_INTELOWL_VERSION=v6.1.0
55
# if you want to use a nfs volume for shared files
66
# NFS_ADDRESS=

integrations/malware_tools_analyzers/Dockerfile

Lines changed: 23 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ RUN npm install [email protected] --global --production \
3030

3131
# Install CAPA
3232
WORKDIR ${PROJECT_PATH}/capa
33-
RUN wget -q https://github.com/mandiant/capa/releases/download/v6.1.0/capa-v6.1.0-linux.zip \
34-
&& unzip capa-v6.1.0-linux.zip \
33+
RUN wget -q https://github.com/mandiant/capa/releases/download/v7.2.0/capa-v7.2.0-linux.zip \
34+
&& unzip capa-v7.2.0-linux.zip \
3535
&& ln -s ${PROJECT_PATH}/capa/capa /usr/local/bin/capa
3636

3737
# Install Floss
@@ -74,28 +74,6 @@ RUN python3 -m venv venv \
7474
&& pip3 install --no-cache-dir --upgrade pip \
7575
&& pip3 install --no-cache-dir -r apkid-requirements.txt
7676

77-
# Install Thug
78-
# https://github.com/buffer/thug/blob/master/docker/Dockerfile
79-
WORKDIR ${PROJECT_PATH}/thug
80-
COPY requirements/thug-requirements.txt ./
81-
RUN python3 -m venv venv \
82-
&& . venv/bin/activate \
83-
&& pip3 install --no-cache-dir --upgrade pip \
84-
&& wget -q https://github.com/cloudflare/stpyv8/releases/download/v11.1.277.14/stpyv8-ubuntu-20.04-python-3.8.zip \
85-
&& unzip stpyv8-ubuntu-20.04-python-3.8.zip \
86-
&& pip3 install --no-cache-dir stpyv8-ubuntu-20.04-3.8/stpyv8-11.1.277.14-cp38-cp38-linux_x86_64.whl \
87-
&& mkdir -p /usr/share/stpyv8 \
88-
&& mv stpyv8-ubuntu-20.04-3.8/icudtl.dat /usr/share/stpyv8 \
89-
&& rm -rf stpyv8-ubuntu-20.04* \
90-
&& git clone https://github.com/buffer/libemu.git && cd libemu && autoreconf -v -i && ./configure && make install && cd .. && rm -rf libemu && ldconfig \
91-
&& pip3 install --no-cache-dir -r thug-requirements.txt \
92-
&& git clone --depth 1 https://github.com/buffer/thug.git \
93-
&& mkdir -p /etc/thug \
94-
&& cp -R thug/thug/conf/* /etc/thug \
95-
&& rm -rf thug \
96-
&& mkdir -p /tmp/thug/logs \
97-
&& chown -R ${USER}:${USER} /tmp/thug/logs
98-
9977
# Install GoReSym
10078
WORKDIR ${PROJECT_PATH}/goresym
10179
RUN wget -q https://github.com/mandiant/GoReSym/releases/download/v2.7.4/GoReSym-linux.zip \
@@ -153,6 +131,27 @@ RUN python3 -m venv venv \
153131
&& chown -R ${USER}:${USER} /root/.semgrep \
154132
&& chmod 711 /root
155133

134+
# Install Thug
135+
# https://github.com/buffer/thug/blob/master/docker/Dockerfile
136+
WORKDIR ${PROJECT_PATH}/thug
137+
COPY requirements/thug-requirements.txt ./
138+
RUN python3 -m venv venv \
139+
&& . venv/bin/activate \
140+
&& pip3 install --no-cache-dir --upgrade pip \
141+
# this is the python 3.9 version. Once you update the base image you should switch to the right version (cp39)
142+
&& wget -q https://github.com/cloudflare/stpyv8/releases/download/v12.7.224.18/stpyv8-12.7.224.18-cp39-cp39-manylinux_2_31_x86_64.whl \
143+
&& pip3 install --no-cache-dir stpyv8-12.7.224.18-cp39-cp39-manylinux_2_31_x86_64.whl \
144+
&& rm stpyv8-12.7.224.18-cp39-cp39-manylinux_2_31_x86_64.whl \
145+
&& mkdir -p /usr/share/stpyv8 \
146+
&& git clone https://github.com/buffer/libemu.git && cd libemu && autoreconf -v -i && ./configure && make install && cd .. && rm -rf libemu && ldconfig \
147+
&& pip3 install --no-cache-dir -r thug-requirements.txt \
148+
&& git clone --depth 1 https://github.com/buffer/thug.git \
149+
&& mkdir -p /etc/thug \
150+
&& cp -R thug/thug/conf/* /etc/thug \
151+
&& rm -rf thug \
152+
&& mkdir -p /tmp/thug/logs \
153+
&& chown -R ${USER}:${USER} /tmp/thug/logs /etc/thug
154+
156155
# prepare fangfrisch installation
157156
COPY crontab /etc/cron.d/crontab
158157
RUN mkdir -m 0770 -p /var/lib/fangfrisch \

integrations/malware_tools_analyzers/app.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,18 +133,16 @@ def intercept_thug_result(context, future: Future) -> None:
133133
by reading the final analysis result from the saved result file
134134
before it is ready to be consumed.
135135
"""
136-
dir_loc = None
137136
# 1. get current result object
138137
res = future.result()
139138
# 2. dir from which we will read final analysis result
140-
dir_name = context.get("read_result_from", None)
141-
if not dir_name:
139+
dir_loc = context.get("read_result_from", None)
140+
if not dir_loc:
142141
res["error"] += ", No specified file to read result from"
143142
if res.get("returncode", -1) == 0:
144143
res["returncode"] = -1
145144
else:
146145
# 3. read saved result file, if it exists
147-
dir_loc = safe_join("/opt/deploy/thug", dir_name)
148146
f_loc = dir_loc + "/analysis/json/analysis.json"
149147
if not os.path.exists(f_loc):
150148
res["error"] += f", result file {f_loc} does not exists."

0 commit comments

Comments
 (0)