Skip to content

Commit 7df2d3a

Browse files
jdorweilerjdorweiler
authored
fix psl parsing (#52)
* fix psl parsing * linter * linter * optional test
1 parent d8f044f commit 7df2d3a

File tree

2 files changed

+155
-5
lines changed

2 files changed

+155
-5
lines changed

‎src/trackers/helpers/url.js

+10-5
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ class ParsedURL extends URL {
1515
}
1616

1717
get domainInfo() {
18-
// extend domainInfo to use PSL
18+
// extend domainInfo to use PSL
1919
if (!this._domainInfo) {
2020
this._domainInfo = parse(this.hostname, {
2121
extractHostname: false,
@@ -25,7 +25,12 @@ class ParsedURL extends URL {
2525
if (!this._domainInfo.isPrivate && pslExtras && pslExtras.privatePSL) {
2626
// get list of possible suffix matches, we can have multiple matches for a single request
2727
// a.example.com and b.a.example.com for a request from 123.b.a.example.com
28-
const suffixMatches = pslExtras.privatePSL.filter(suffix => this._domainInfo.hostname.match(suffix))
28+
// check that suffix is preceded by a dot or is at the beginning of the hostname
29+
const suffixMatches = pslExtras.privatePSL.filter(suffix => {
30+
const escapedSuffix = suffix.replace('.', '\\.')
31+
const regex = new RegExp(`(^|\\.)${escapedSuffix}$`)
32+
return regex.test(this._domainInfo.hostname)
33+
})
2934

3035
// reformat domainInfo to make this request look like a private domain
3136
if (suffixMatches && suffixMatches.length) {
@@ -34,13 +39,13 @@ class ParsedURL extends URL {
3439
const suffix = suffixMatches.reduce((l,s) => {
3540
return l.length >= s.length ? l : s
3641
})
37-
42+
3843
// Array of subdomain after removing suffix from hostname
39-
const splitSubdomain = this._domainInfo.hostname.replace(suffix, '').replace(/\.$/,'').split('.')
44+
const splitSubdomain = this._domainInfo.hostname.replace(new RegExp(`\\.?${suffix}$`), '').split('.')
4045
const domainWithoutSuffix = splitSubdomain.pop()
4146

4247
this._domainInfo.publicSuffix = suffix
43-
this._domainInfo.domain = `${domainWithoutSuffix}.${suffix}`
48+
this._domainInfo.domain = domainWithoutSuffix ? `${domainWithoutSuffix}.${suffix}` : suffix
4449
this._domainInfo.domainWithoutSuffix = domainWithoutSuffix
4550
this._domainInfo.subdomain = splitSubdomain.join('.')
4651
this._domainInfo.isPrivate = true

‎test/custom-psl-paring.test.js

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
const assert = require('assert');
2+
const URL = require('../src/trackers/helpers/url.js');
3+
const sharedData = require('../src/trackers/helpers/sharedData')
4+
5+
const testCases = [
6+
{
7+
input: "media-rockstargames-com.akamaized.net",
8+
expectedOutput: {
9+
publicSuffix: "akamaized.net",
10+
domain: "media-rockstargames-com.akamaized.net",
11+
domainWithoutSuffix: "media-rockstargames-com",
12+
subdomain: "",
13+
isPrivate: true
14+
}
15+
},
16+
{
17+
input: "sub.media-rockstargames-com.akamaized.net",
18+
expectedOutput: {
19+
publicSuffix: "akamaized.net",
20+
domain: "media-rockstargames-com.akamaized.net",
21+
domainWithoutSuffix: "media-rockstargames-com",
22+
subdomain: "sub",
23+
isPrivate: true
24+
}
25+
},
26+
{
27+
input: "example.akamaihd.net",
28+
expectedOutput: {
29+
publicSuffix: "akamaihd.net",
30+
domain: "example.akamaihd.net",
31+
domainWithoutSuffix: "example",
32+
subdomain: "",
33+
isPrivate: true
34+
}
35+
},
36+
{
37+
input: "a.b.c.d.e.example.akamaihd.net",
38+
expectedOutput: {
39+
publicSuffix: "akamaihd.net",
40+
domain: "example.akamaihd.net",
41+
domainWithoutSuffix: "example",
42+
subdomain: "a.b.c.d.e",
43+
isPrivate: true
44+
}
45+
},
46+
{
47+
input: "example.com",
48+
expectedOutput: {
49+
publicSuffix: "com",
50+
domain: "example.com",
51+
domainWithoutSuffix: "example",
52+
subdomain: "",
53+
isPrivate: false
54+
}
55+
},
56+
{
57+
input: "sub.test.edgekey-staging.net",
58+
expectedOutput: {
59+
publicSuffix: "test.edgekey-staging.net",
60+
domain: "sub.test.edgekey-staging.net",
61+
domainWithoutSuffix: "sub",
62+
subdomain: "",
63+
isPrivate: true
64+
}
65+
},
66+
{
67+
input: "bucket.s3.us-west-2.amazonaws.com",
68+
expectedOutput: {
69+
publicSuffix: "s3.us-west-2.amazonaws.com",
70+
domain: "bucket.s3.us-west-2.amazonaws.com",
71+
domainWithoutSuffix: "bucket",
72+
subdomain: "",
73+
isPrivate: true
74+
}
75+
},
76+
{
77+
input: "example.ssl.global.fastly.net",
78+
expectedOutput: {
79+
publicSuffix: "ssl.global.fastly.net",
80+
domain: "example.ssl.global.fastly.net",
81+
domainWithoutSuffix: "example",
82+
subdomain: "",
83+
isPrivate: true
84+
}
85+
},
86+
{
87+
input: "example.x.incapdns.net",
88+
expectedOutput: {
89+
publicSuffix: "x.incapdns.net",
90+
domain: "example.x.incapdns.net",
91+
domainWithoutSuffix: "example",
92+
subdomain: "",
93+
isPrivate: true
94+
}
95+
},
96+
{
97+
input: "example.trafficmanager.net",
98+
expectedOutput: {
99+
publicSuffix: "trafficmanager.net",
100+
domain: "example.trafficmanager.net",
101+
domainWithoutSuffix: "example",
102+
subdomain: "",
103+
isPrivate: true
104+
}
105+
},
106+
{
107+
input: "akamaized.net",
108+
expectedOutput: {
109+
publicSuffix: "akamaized.net",
110+
domain: "akamaized.net",
111+
domainWithoutSuffix: "",
112+
subdomain: "",
113+
isPrivate: true
114+
}
115+
},
116+
{
117+
input: "example.com.akamaized.net",
118+
expectedOutput: {
119+
publicSuffix: "com.akamaized.net",
120+
domain: "example.com.akamaized.net",
121+
domainWithoutSuffix: "example",
122+
subdomain: "",
123+
isPrivate: true
124+
}
125+
}
126+
];
127+
128+
describe('PSL domain parsing', () => {
129+
describe('parse domain with custom PSL', () => {
130+
testCases.forEach(({ input, expectedOutput }) => {
131+
it(`should correctly parse ${input}`, () => {
132+
if (!sharedData.config.pslExtras) {
133+
it.skip('No custom PSL data provided')
134+
} else {
135+
const result = new URL('https://' + input).domainInfo
136+
assert.strictEqual(result.publicSuffix, expectedOutput.publicSuffix)
137+
assert.strictEqual(result.domain, expectedOutput.domain)
138+
assert.strictEqual(result.domainWithoutSuffix, expectedOutput.domainWithoutSuffix)
139+
assert.strictEqual(result.subdomain, expectedOutput.subdomain)
140+
assert.strictEqual(result.isPrivate, expectedOutput.isPrivate)
141+
}
142+
});
143+
});
144+
});
145+
});

0 commit comments

Comments
 (0)