forked from laironald/PythonBase
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathsenAdd.py~
135 lines (122 loc) · 6.29 KB
/
senAdd.py~
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
import csv, re, sqlite3, types
#colors = [['Orange Red', '#FF4500'], ['Gold', '#FFD700'], ['Orange', '#FFA500'], ['Tomato', '#FF6347'], ['Deep Pink', '#FF1493'], ['Magenta', '#FF00FF'], ['Salmon', '#FA8072'], ['Sandy Brown', '#F4A460'], ['Khaki', '#F0E68C'], ['Light Cyan', '#E0FFFF'], ['Yellow Green', '#9ACD32'], ['Pale Green', '#98FB98'], ['Dark Violet', '#9400D3'], ['Medium Purple', '#9370DB'], ['Sky Blue', '#87CEEB'], ['Crimson', '#DC143C'], ['Olive', '#808000'], ['Slate Gray', '#708090'], ['Dim Gray', '#696969'], ['Dark Olive Green', '#556B2F'], ['Steel Blue', '#4682B4'], ['Aqua', '#00FFFF'], ['Lime', '#00FF00'], ['Deep Sky Blue', '#00BFFF'], ['Teal', '#008080'], ['Dark Green', '#006400'], ['Medium Blue', '#0000CD'], ['Spring Green', '#00FF7F'], ['Violet', '#EE82EE'], ['Light Blue', '#ADD8E6'], ['Light Grey', '#D3D3D3'], ['Light Pink', '#FFB6C1'], ['Light Salmon', '#FFA07A'], ['Light Steel Blue', '#B0C4DE']]
colors = [['Orange Red', '#FF4500'], ['Gold', '#FFD700'], ['Orange', '#FFA500'], ['Tomato', '#FF6347'], ['Deep Pink', '#FF1493'], ['Magenta', '#FF00FF'], ['Salmon', '#FA8072'], ['Sandy Brown', '#F4A460'], ['Khaki', '#F0E68C'], ['Yellow Green', '#9ACD32'], ['Pale Green', '#98FB98'], ['Dark Violet', '#9400D3'], ['Medium Purple', '#9370DB'], ['Crimson', '#DC143C'], ['Olive', '#808000'], ['Slate Gray', '#708090'], ['Dim Gray', '#696969'], ['Dark Olive Green', '#556B2F'], ['Steel Blue', '#4682B4'], ['Lime', '#00FF00'], ['Deep Sky Blue', '#00BFFF'], ['Teal', '#008080'], ['Dark Green', '#006400'], ['Medium Blue', '#0000CD'], ['Spring Green', '#00FF7F'], ['Light Grey', '#D3D3D3'], ['Light Pink', '#FFB6C1'], ['Light Salmon', '#FFA07A']]
def colGrad(col1="#000000", col2="#FF0000", pct=50):
r = hex((int(col2[1:3], 16) - int(col1[1:3], 16))*pct/100 + int(col1[1:3], 16))[-2:]
g = hex((int(col2[3:5], 16) - int(col1[3:5], 16))*pct/100 + int(col1[3:5], 16))[-2:]
b = hex((int(col2[5:7], 16) - int(col1[5:7], 16))*pct/100 + int(col1[5:7], 16))[-2:]
return ("#%s%s%s" % (r, g, b)).replace("x", "0")
def csvInput(fname):
return [x for x in csv.reader(open(fname, "rb"))]
def ascit(x, strict=True):
import unicodedata
x = x.upper()
#Solves that {UMLAUT OVER (A)}
x = re.sub(r"[{].*?[(].*?[)].*?[}]", lambda(x):re.findall("[(](.*?)[)]", x.group())[0], x)
#remove space(s) + punctuation
x = re.sub(r" *?[,|-] *?", lambda(x):re.findall(r"[,|-]", x.group())[0], x)
if strict:
#remove stuff in between (), {}
x = re.sub(r"[(].*?[)]|[{].*?[}]", "", x)
#remove periods, ampersand, etc
##x = re.sub(r"[!@#$%^&*.,(){}]", "", x)
x = re.sub(r"[^A-Za-z0-9 ]", " ", x)
x = re.sub(r" +", " ", x)
#remove duplicates
x = re.sub(r"[ ,|-]{2,}", lambda(x):re.findall(r"[ ,|-]", x.group())[0], x)
#remove all unicode
x = unicodedata.normalize('NFKD', unicode(x)).encode('ascii', 'ignore')
return x.strip()
def jarow(s1,s2):
try:
s1 = s1.upper()
s2 = s2.upper()
if s1==s2:
return 1.0
if s1=="" or s2=="":
return 0.0
short, long = len(s1)>len(s2) and [s2, s1] or [s1, s2]
for l in range(0, min(5, len(short))):
if short[l] != long[l]:
break
mtch = ""
mtch2=[]
dist = len(long)/2-1
m = 0.0
for i, x in enumerate(long):
jx, jy = (lambda x,y: x==y and (x,y+1) or (x,y))(max(0, i-dist), min(len(short), i+dist))
for j in range(jx, jy):
if j<len(short) and x == short[j]:
m+=1
mtch+=x
mtch2.extend([[j,x]])
short=short[:j]+"*"+short[min(len(short), j+1):]
break
mtch2 = "".join(x[1] for x in sorted(mtch2))
t = 0.0
for i in range(0, len(mtch)):
if mtch[i]!=mtch2[i]:
t += 0.5
d = 0.1
# this is the jaro-distance
if m==0:
d_j = 0
else:
d_j = 1/3.0 * ((m/len(short)) + (m/len(long)) + ((m - t)/m))
return d_j + (l * d * (1 - d_j))
except:
return 0
#SPECIFICALLY FOR SEN
def invComp(s1, s2):
sX = re.split(r" ?[.,] ?", s1)
sY = re.split(r" ?[.,] ?", s2)
if len(sX)==2 and len(sY)==2:
if sX[1].find(sY[1])>-1 or sY[1].find(sX[1])>-1:
llen = [min(len(sX[i]), len(sY[i])) for i in range(0,2)]
return sum([jarow(sX[i], sY[i])*llen[i] for i in range(0,2)])/sum(llen)
else:
return 0.0
else:
return jarow(s1, s2)
def invXMtch(st):
sts = st.split("|")
ok = False
for i in range(0,len(sts)-1):
for j in range(i+1, len(sts)):
if invComp(sts[i], sts[j])==0.0:
ok = True
break
return ok
#---------------------
class uQvals:
def __init__(self):
self.dList=[]
def step(self, value):
self.dList.append(value.upper())
def finalize(self):
out = list(set(self.dList))
return len(out)>1 and "|".join(out) or ""
def quickSQL(c, data, table="", header=False):
if table=="":
table = "debug%d" % len([x[0] for x in c.execute("SELECT tbl_name FROM sqlite_master WHERE type='table' order by tbl_name") if len(re.findall(r"debug[0-9]+", x[0]))>0])
vTp = {types.StringType:"VARCHAR", types.UnicodeType:"VARCHAR", types.IntType:"INTEGER", types.FloatType: "REAL"}
if header==False:
c.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (table, ", ".join(["v%d %s" % (i, vTp[type(x)]) for i,x in enumerate(data[0])])))
c.executemany("INSERT INTO %s VALUES (%s)" % (table, ", ".join(["?"]*len(data[0]))), data)
else:
c.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (table, ", ".join(["%s %s" % (data[0][i], vTp[type(x)]) for i,x in enumerate(data[1])])))
c.executemany("INSERT INTO %s VALUES (%s)" % (table, ", ".join(["?"]*len(data[0]))), data[1:])
def flatten(l, ltypes=(list, tuple)):
ltype = type(l)
l = list(l)
i = 0
while i < len(l):
while isinstance(l[i], ltypes):
if not l[i]:
l.pop(i)
i -= 1
break
else:
l[i:i + 1] = l[i]
i += 1
return ltype(l)