Skip to content

Commit 53aa997

Browse files
ashtumashtum
committed
Extract search records for learn sections
1 parent 3c1ee1a commit 53aa997

8 files changed

Lines changed: 157 additions & 18 deletions

File tree

‎.github/workflows/index_on_algolia.yml‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
name: Index on Algolia
22

33
on:
4+
pull_request:
45
push:
56
branches: [develop, ci-*]
67
paths:
@@ -26,18 +27,28 @@ jobs:
2627
- name: Install dependencies
2728
run: pip install -r requirements.txt
2829

30+
- name: Clone and build website-v2-docs
31+
run: |
32+
git clone --depth=1 --branch=master https://github.com/boostorg/website-v2-docs.git ../website-v2-docs
33+
cd ../website-v2-docs
34+
./build.sh
35+
2936
- name: Download and extract boost release archive
3037
run: |
31-
BOOST_VERSION=$(sed -n 's/.*version: "\(.*\)"/\1/p' config/config.yaml)
32-
BOOST_VERSION_MAIN=$(echo $BOOST_VERSION | sed -E 's/([0-9]+)_([0-9]+)_([0-9]+)(.*)/\1.\2.\3/g')
33-
wget --no-verbose https://boostorg.jfrog.io/artifactory/main/release/$BOOST_VERSION_MAIN/source/boost_$BOOST_VERSION.tar.gz
34-
tar -xzf boost_$BOOST_VERSION.tar.gz -C ../
38+
mkdir ../boost_1_85_0
39+
# BOOST_VERSION=$(sed -n 's/.*version: "\(.*\)"/\1/p' config/config.yaml)
40+
# BOOST_VERSION_MAIN=$(echo $BOOST_VERSION | sed -E 's/([0-9]+)_([0-9]+)_([0-9]+)(.*)/\1.\2.\3/g')
41+
# wget --no-verbose https://boostorg.jfrog.io/artifactory/main/release/$BOOST_VERSION_MAIN/source/boost_$BOOST_VERSION.tar.gz
42+
# tar -xzf boost_$BOOST_VERSION.tar.gz -C ../
43+
44+
- name: Extract learn records
45+
run: python -m gecko.extract_learn_records
3546

36-
- name: Extract records
37-
run: python -m gecko.extract_records
47+
# - name: Extract libraries records
48+
# run: python -m gecko.extract_libraries_records
3849

39-
- name: Check validity of records
40-
run: python -m gecko.sanitizer check
50+
# - name: Check validity of records
51+
# run: python -m gecko.sanitizer check
4152

4253
- name: Index on Algolia
4354
env:

algolia_records/.gitignore renamed to algolia_records/learn/.gitignore

File renamed without changes.

‎algolia_records/libraries/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.json

‎config/config.yaml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
boost:
2-
version: "1_85_0"
3-
root: "../boost_1_85_0"
2+
version: '1_85_0'
3+
root: '../boost_1_85_0'
4+
5+
website-v2-docs:
6+
root: '../website-v2-docs/build'
7+
sections:
8+
- key: 'contributor-guide'
9+
name: 'Contributor Guide'
10+
- key: 'formal-reviews'
11+
name: 'Formal Reviews'
12+
- key: 'user-guide'
13+
name: 'User Guide'
414

515
algolia:
616
app-id: D7O1MLLTAF

‎gecko/config.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,19 @@
77
'version': str,
88
'root': os.path.exists
99
},
10+
'website-v2-docs': {
11+
'root': os.path.exists,
12+
'sections': [
13+
{
14+
'key': str,
15+
'name': str
16+
}
17+
]
18+
},
1019
'algolia': {
1120
'app-id': str,
1221
'api-key': str,
13-
'settings':dict
22+
'settings': dict
1423
},
1524
'crawlers': [
1625
{

‎gecko/extract_learn_records.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import re
2+
import json
3+
from pathlib import Path
4+
from urllib.parse import urljoin
5+
from bs4 import BeautifulSoup, Tag
6+
7+
from .crawlers.helpers import has_class
8+
from .config import config
9+
10+
11+
class AntoraDoc():
12+
def crawl(self, doc_root: Path) -> dict:
13+
sections = {}
14+
doc_root = doc_root.resolve()
15+
16+
for file_path in doc_root.rglob('*.html'):
17+
with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:
18+
soup = BeautifulSoup(file.read(), 'html.parser')
19+
20+
lvls = []
21+
for link in soup.select('body nav.breadcrumbs ul li a'):
22+
lvls = lvls + [{'title': link.text, 'path': urljoin(str(file_path), link.get('href'))}]
23+
24+
sect1 = soup.select_one('body article.doc')
25+
if sect1:
26+
self._extract_section_n(str(file_path), sections, sect1, lvls)
27+
28+
return sections
29+
30+
def _extract_section_n(self, file_path: str, sections: dict, sect: Tag, lvls: list = []):
31+
header = sect.select_one('h1, h2, h3, h4, h5, h6')
32+
33+
if header.name == 'h1':
34+
path = file_path
35+
else:
36+
title = header.text
37+
path = file_path + '#' + header.get('id')
38+
lvls = lvls + [{'title': title, 'path': path}]
39+
40+
if header.find_next_sibling() and has_class(header.find_next_sibling(), 'sectionbody'):
41+
siblings = header.find_next_sibling().find_all(recursive=False)
42+
else:
43+
siblings = header.next_siblings
44+
45+
content = ''
46+
for sibling in siblings:
47+
if isinstance(sibling, Tag) and sibling.has_attr('class') and len([i for i in sibling.get('class') if i.startswith('sect')]) > 0:
48+
self._extract_section_n(file_path, sections, sibling, lvls)
49+
continue
50+
content += sibling.get_text() + ' '
51+
52+
sections[path] = {'content': content, 'lvls': lvls}
53+
54+
55+
def create_algolia_records(section_key: str, section_name: str, doc_root: Path, sections: dict):
56+
doc_root = doc_root.resolve()
57+
records = []
58+
59+
for _, section in sections.items():
60+
for lvl in section['lvls']:
61+
lvl['path'] = lvl['path'].replace(str(doc_root) + '/', '')
62+
63+
records.append({
64+
'type': 'content',
65+
'section_key': section_key,
66+
'section_name': section_name,
67+
'content': re.sub(r'\s+', ' ', section['content']).strip(),
68+
'weight': {
69+
'pageRank': 0,
70+
'level': 100 - len(section['lvls']) * 10,
71+
'position': 0
72+
},
73+
'path': section['lvls'][-1]['path'] if len(section['lvls']) > 0 else None,
74+
'hierarchy': {
75+
'lvl0': section['lvls'][0] if len(section['lvls']) > 0 else None,
76+
'lvl1': section['lvls'][1] if len(section['lvls']) > 1 else None,
77+
'lvl2': section['lvls'][2] if len(section['lvls']) > 2 else None,
78+
'lvl3': section['lvls'][3] if len(section['lvls']) > 3 else None,
79+
'lvl4': section['lvls'][4] if len(section['lvls']) > 4 else None,
80+
'lvl5': section['lvls'][5] if len(section['lvls']) > 5 else None,
81+
'lvl6': section['lvls'][6] if len(section['lvls']) > 6 else None
82+
}})
83+
84+
with open('./algolia_records/learn/' + section_key + '.json', 'w', encoding='utf-8') as outfile:
85+
json.dump(records, outfile, indent=4)
86+
87+
88+
if __name__ == "__main__":
89+
crawler = AntoraDoc()
90+
91+
for section in config['website-v2-docs']['sections']:
92+
sections = crawler.crawl(Path(config['website-v2-docs']['root']) / section['key'])
93+
create_algolia_records(section['key'],
94+
section['name'],
95+
Path(config['website-v2-docs']['root']),
96+
sections)

gecko/extract_records.py renamed to gecko/extract_libraries_records.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def create_algolia_records(library_key: str, sections: dict, boost_root: str):
7575
'lvl6': section['lvls'][6] if len(section['lvls']) > 6 else None
7676
}})
7777

78-
with open('./algolia_records/' + library_key.replace('/', '_') + '.json', 'w', encoding='utf-8') as outfile:
78+
with open('./algolia_records/libraries/' + library_key.replace('/', '_') + '.json', 'w', encoding='utf-8') as outfile:
7979
json.dump(records, outfile, indent=4)
8080

8181

‎gecko/index_on_algolia.py‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,19 @@
99
client = SearchClient.create(config['algolia']['app-id'], config['algolia']['api-key'])
1010

1111
print('Initializing {} index ...'.format(config['boost']['version']))
12-
index = client.init_index(config['boost']['version'])
12+
libraries_index = client.init_index(config['boost']['version'])
1313

1414
print('Setting settings for {} index ...'.format(config['boost']['version']))
15-
index.set_settings(config['algolia']['settings'])
15+
libraries_index.set_settings(config['algolia']['settings'])
1616

17-
for path in Path('./algolia_records').glob('*.json'):
17+
for path in Path('./algolia_records/libraries').glob('*.json'):
1818
print('uploading records for {}...'.format(path.stem))
1919

2020
with open(path, 'r', encoding='utf-8') as f:
2121
records = json.load(f)
2222

2323
# Delete the existing records for this library.
24-
index.delete_by({'filters': 'library_key:{}'.format(records[0]['library_key'])})
24+
libraries_index.delete_by({'filters': 'library_key:{}'.format(records[0]['library_key'])})
2525

2626
# Split long documents into smaller parts.
2727
for record in records:
@@ -34,5 +34,17 @@
3434
records = [record for record in records if not (
3535
record['content'] == '' and not record['hierarchy']['lvl0'])]
3636

37-
# TODO instead of using autoGenerateObjectIDIfNotExist we might create a hash out of hierarchy items
38-
index.save_objects(records, {'autoGenerateObjectIDIfNotExist': True})
37+
libraries_index.save_objects(records, {'autoGenerateObjectIDIfNotExist': True})
38+
39+
learn_index = client.init_index('learn')
40+
41+
for path in Path('./algolia_records/learn').glob('*.json'):
42+
print('uploading records for {}...'.format(path.stem))
43+
44+
with open(path, 'r', encoding='utf-8') as f:
45+
records = json.load(f)
46+
47+
# Delete the existing records for this library.
48+
libraries_index.delete_by({'filters': 'section_key:{}'.format(records[0]['section_key'])})
49+
50+
libraries_index.save_objects(records, {'autoGenerateObjectIDIfNotExist': True})

0 commit comments

Comments
 (0)