Skip to content

Commit 9964aeb

Browse files
HolgerHuoHolgerHuo
authored
Merge pull request #17 from AllanChain/feat/references
feat(cite): enable reference autocomplete
2 parents e501310 + b796bc0 commit 9964aeb

File tree

11 files changed

+385
-0
lines changed

11 files changed

+385
-0
lines changed

‎develop/dev.env‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@ WEBPACK_HOST=webpack
1818
WEB_API_PASSWORD=overleaf
1919
WEB_API_USER=overleaf
2020
WEB_HOST=web
21+
REFERENCES_HOST=references

‎develop/docker-compose.dev.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,17 @@ services:
7979
- ../services/history-v1/knexfile.js:/overleaf/services/history-v1/knexfile.js
8080
- ../services/history-v1/migrations:/overleaf/services/history-v1/migrations
8181

82+
references:
83+
command: ["node", "--watch", "app.js"]
84+
environment:
85+
- NODE_OPTIONS=--inspect=0.0.0.0:9229
86+
ports:
87+
- "127.0.0.1:9236:9229"
88+
volumes:
89+
- ../services/references/app:/overleaf/services/references/app
90+
- ../services/references/config:/overleaf/services/references/config
91+
- ../services/references/app.js:/overleaf/services/references/app.js
92+
8293
notifications:
8394
command: ["node", "--watch", "app.js"]
8495
environment:

‎develop/docker-compose.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@ services:
9696
volumes:
9797
- mongo-data:/data/db
9898

99+
references:
100+
build:
101+
context: ..
102+
dockerfile: services/references/Dockerfile
103+
env_file:
104+
- dev.env
105+
99106
notifications:
100107
build:
101108
context: ..
@@ -161,6 +168,7 @@ services:
161168
- filestore
162169
- history-v1
163170
- notifications
171+
- references
164172
- project-history
165173
- real-time
166174
- spelling

‎server-ce/runit/references-overleaf/run‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
NODE_PARAMS=""
4+
if [ "$DEBUG_NODE" == "true" ]; then
5+
echo "running debug - references"
6+
NODE_PARAMS="--inspect=0.0.0.0:30060"
7+
fi
8+
9+
NODE_CONFIG_DIR=/overleaf/services/references/config exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/references/app.js >> /var/log/overleaf/references.log 2>&1

‎services/references/Dockerfile‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
FROM node:18.20.2 AS base
2+
3+
WORKDIR /overleaf/services/references
4+
5+
# Google Cloud Storage needs a writable $HOME/.config for resumable uploads
6+
# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream)
7+
RUN mkdir /home/node/.config && chown node:node /home/node/.config
8+
9+
FROM base AS app
10+
11+
COPY package.json package-lock.json /overleaf/
12+
COPY services/references/package.json /overleaf/services/references/
13+
COPY libraries/ /overleaf/libraries/
14+
COPY patches/ /overleaf/patches/
15+
16+
RUN cd /overleaf && npm ci --quiet
17+
18+
COPY services/references/ /overleaf/services/references/
19+
20+
FROM app
21+
USER node
22+
23+
CMD ["node", "--expose-gc", "app.js"]
24+

‎services/references/app.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import '@overleaf/metrics/initialize.js'
2+
3+
import express from 'express'
4+
import Settings from '@overleaf/settings'
5+
import logger from '@overleaf/logger'
6+
import metrics from '@overleaf/metrics'
7+
import ReferencesAPIController from './app/js/ReferencesAPIController.js'
8+
import bodyParser from 'body-parser'
9+
10+
const app = express()
11+
metrics.injectMetricsRoute(app)
12+
13+
app.use(bodyParser.json({ limit: '2mb' }))
14+
app.use(metrics.http.monitor(logger))
15+
16+
app.post('/project/:project_id/index', ReferencesAPIController.index)
17+
app.get('/status', (req, res) => res.send({ status: 'references api is up' }))
18+
19+
const settings =
20+
Settings.internal && Settings.internal.references
21+
? Settings.internal.references
22+
: undefined
23+
const host = settings && settings.host ? settings.host : 'localhost'
24+
const port = settings && settings.port ? settings.port : 3006
25+
26+
logger.debug('Listening at', { host, port })
27+
28+
const server = app.listen(port, host, function (error) {
29+
if (error) {
30+
throw error
31+
}
32+
logger.info({ host, port }, 'references HTTP server starting up')
33+
})
34+
35+
process.on('SIGTERM', () => {
36+
server.close(() => {
37+
logger.info({ host, port }, 'references HTTP server closed')
38+
metrics.close()
39+
})
40+
})

‎services/references/app/js/BibParser.js‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
"use strict";
2+
3+
// Grammar implemented here:
4+
// bibtex -> (string | entry)*;
5+
// string -> '@STRING' kv_left key_equals_value kv_right;
6+
// entry -> '@' key kv_left key ',' key_value_list kv_right;
7+
// key_value_list -> key_equals_value (',' key_equals_value)* ','?;
8+
// key_equals_value -> key '=' value;
9+
// value -> value_quotes | value_braces | key;
10+
// value_quotes -> '"' .*? '"'; // not quite
11+
// value_braces -> '{' .*? '"'; // not quite
12+
// kv_left -> '(' | '{'
13+
// kv_right -> ')' | '}'
14+
function BibtexParser() {
15+
this._entries = {};
16+
this._comments = [];
17+
this._strings = {};
18+
this.input = '';
19+
this.config = {
20+
upperKeys: false
21+
};
22+
this._pos = 0;
23+
var pairs = {
24+
'{': '}',
25+
'(': ')',
26+
'"': '"'
27+
};
28+
var regs = {
29+
atKey: /@([a-zA-Z0-9_:\\./-]+)\s*/,
30+
enLeft: /^([\{\(])\s*/,
31+
enRight: function enRight(left) {
32+
return new RegExp("^(\\".concat(pairs[left], ")\\s*"));
33+
},
34+
entryId: /^\s*([^@={}",\s]+)\s*,\s*/,
35+
key: /^([a-zA-Z0-9_:\\./-]+)\s*=\s*/,
36+
vLeft: /^([\{"])\s*/,
37+
vRight: function vRight(left) {
38+
return new RegExp("^(\\".concat(pairs[left], ")\\s*"));
39+
},
40+
inVLeft: /^(\{)\s*/,
41+
inVRight: function inVRight(left) {
42+
return new RegExp("^(\\".concat(pairs[left], ")\\s*"));
43+
},
44+
value: /^[\{"]((?:[^\{\}]|\n)*?(?:(?:[^\{\}]|\n)*?\{(?:[^\{\}]|\n)*?\})*?(?:[^\{\}]|\n)*?)[\}"]\s*,?\s*/,
45+
word: /^([^\{\}"\s]+)\s*/,
46+
comma: /^(,)\s*/,
47+
quota: /^(")\s*/
48+
};
49+
50+
this.setInput = function (t) {
51+
this.input = t;
52+
};
53+
54+
this.matchFirst = function (reg) {
55+
var notMove = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
56+
var result = this.input.slice(this._pos).match(reg);
57+
58+
if (result) {
59+
if (!notMove) {
60+
// console.log("!@#!@#", result[1]);
61+
this._pos += result.index + result[0].length;
62+
}
63+
64+
return {
65+
success: true,
66+
text: result[1],
67+
index: result.index,
68+
step: result[0].length
69+
};
70+
} else {
71+
return {
72+
success: false
73+
};
74+
}
75+
};
76+
77+
this.assert = function (obj) {
78+
for (var key in obj) {
79+
if (obj[key] === undefined) {
80+
throw "[BibParser:ERROR] ".concat(key, " not found at ").concat(this._pos);
81+
}
82+
}
83+
};
84+
85+
this.getValue = function () {
86+
var stack = [];
87+
var values = [];
88+
89+
var _this$matchFirst = this.matchFirst(regs.vLeft),
90+
vLeft = _this$matchFirst.text;
91+
92+
this.assert({
93+
vLeft: vLeft
94+
});
95+
stack.push(vLeft);
96+
97+
while (stack.length > 0) {
98+
if (this.matchFirst(regs.inVLeft, true).success) {
99+
var _this$matchFirst2 = this.matchFirst(regs.inVLeft),
100+
inVLeft = _this$matchFirst2.text;
101+
102+
stack.push(inVLeft);
103+
values.push(inVLeft);
104+
} else if (this.matchFirst(regs.inVRight(stack[stack.length - 1]), true).success) {
105+
values.push(this.matchFirst(regs.inVRight(stack[stack.length - 1])).text);
106+
stack.pop();
107+
} else if (this.matchFirst(regs.word, true).success) {
108+
values.push(this.matchFirst(regs.word).text);
109+
} else if (this.matchFirst(regs.quota, true).success) {
110+
values.push(this.matchFirst(regs.quota).text);
111+
} else {
112+
throw "[BibParser:ERROR] stack overflow at ".concat(this._pos);
113+
}
114+
}
115+
116+
values.pop();
117+
this.matchFirst(regs.comma);
118+
return values;
119+
};
120+
121+
this.string = function () {
122+
var _this$matchFirst3 = this.matchFirst(regs.key),
123+
key = _this$matchFirst3.text;
124+
125+
this.assert({
126+
key: key
127+
});
128+
129+
var _this$matchFirst4 = this.matchFirst(regs.value),
130+
value = _this$matchFirst4.text;
131+
132+
this.assert({
133+
value: value
134+
});
135+
this._strings[key] = value;
136+
};
137+
138+
this.preamble = function () {};
139+
140+
this.comment = function () {};
141+
142+
this.entry = function (head) {
143+
var _this$matchFirst5 = this.matchFirst(regs.entryId),
144+
entryId = _this$matchFirst5.text;
145+
146+
this.assert({
147+
entryId: entryId
148+
});
149+
var entry = {};
150+
151+
while (this.matchFirst(regs.key, true).success) {
152+
var _this$matchFirst6 = this.matchFirst(regs.key),
153+
key = _this$matchFirst6.text;
154+
155+
var value = this.getValue();
156+
entry[key] = value.join(' '); // if(key === 'author'){
157+
// const {text:value} = this.matchFirst(regs.value);
158+
// this.assert({value});
159+
// entry[key] = value;
160+
// } else {
161+
// const {text:value} = this.matchFirst(regs.value);
162+
// this.assert({value});
163+
// entry[key] = value;
164+
// }
165+
}
166+
167+
entry.$type = head;
168+
this._entries[entryId] = entry;
169+
};
170+
171+
this.parse = function () {
172+
while (this.matchFirst(regs.atKey, true).success) {
173+
var _this$matchFirst7 = this.matchFirst(regs.atKey),
174+
head = _this$matchFirst7.text;
175+
176+
var _this$matchFirst8 = this.matchFirst(regs.enLeft),
177+
enLeft = _this$matchFirst8.text;
178+
179+
this.assert({
180+
enLeft: enLeft
181+
});
182+
183+
if (head.toUpperCase() == 'STRING') {
184+
this.string();
185+
} else if (head.toUpperCase() == 'PREAMBLE') {
186+
this.preamble();
187+
} else if (head.toUpperCase() == 'COMMENT') {
188+
this.comment();
189+
} else {
190+
this.entry(head);
191+
}
192+
193+
var _this$matchFirst9 = this.matchFirst(regs.enRight(enLeft)),
194+
enRight = _this$matchFirst9.text;
195+
196+
this.assert({
197+
enRight: enRight
198+
});
199+
}
200+
};
201+
} //Runs the parser
202+
203+
204+
export function bibParse(input) {
205+
var b = new BibtexParser();
206+
b.setInput(input);
207+
b.parse();
208+
return b._entries;
209+
}

‎services/references/app/js/ReferencesAPIController.js‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import logger from '@overleaf/logger'
2+
import { bibParse } from './BibParser.js'
3+
4+
// req: { allUrls: string[], fullIndex: boolean }
5+
// res: { keys: string[]}
6+
export default {
7+
index(req, res) {
8+
const { docUrls, fullIndex } = req.body;
9+
Promise.all(docUrls.map(async (docUrl) => {
10+
try {
11+
const response = await fetch(docUrl);
12+
if (!response.ok) {
13+
throw new Error(`HTTP error! status: ${response.status}`);
14+
}
15+
return response.text();
16+
} catch (error) {
17+
logger.error({ error }, "Failed to fetch document from URL: " + docUrl);
18+
return null;
19+
}
20+
})).then((responses) => {
21+
const keys = [];
22+
for (const body of responses) {
23+
if (!body) continue
24+
try {
25+
const result = bibParse(body);
26+
const resultKeys = Object.keys(result);
27+
keys.push(...resultKeys);
28+
} catch(error) {
29+
logger.error({error}, "skip the file.")
30+
}
31+
}
32+
logger.info({ keys }, "all keys");
33+
res.send({ keys })
34+
})
35+
}
36+
}

‎services/references/config/settings.defaults.cjs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
module.exports = {
2+
internal: {
3+
references: {
4+
port: 3006,
5+
host: process.env.REFERENCES_HOST || '127.0.0.1',
6+
},
7+
},
8+
}
9+

0 commit comments

Comments
 (0)