Skip to content

Commit 8662297

Browse files
erhanbaserhanbas
committed
initial commit with files
1 parent fe46a37 commit 8662297

8 files changed

+458
-0
lines changed

‎.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "common"]
2+
path = common
3+
url = https://github.com/erhanbas/common.git

‎common

Submodule common added at 2c5432e

‎pyglass.so

12.4 MB
Binary file not shown.

‎singleThreadedCacher.py

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#! python2
2+
import sys
3+
import time
4+
from os import path, listdir, makedirs
5+
import numpy as np
6+
from skimage import io
7+
import pyglass
8+
9+
default_max_intensity = 15000.0
10+
default_num_worker_threads = 12
11+
default_max_queue_size = 12
12+
13+
14+
def worker(queue, max_intensity, input_root_path, output_root_path):
15+
while len(queue) > 0:
16+
block = queue.pop(0)
17+
input_folder = path.join(input_root_path, block)
18+
output_folder = path.join(output_root_path, block)
19+
try:
20+
makedirs(output_folder)
21+
except OSError:
22+
if not path.isdir(output_folder):
23+
raise
24+
valid_folders = [str(n) for n in range(1, 9)]
25+
children = [x for x in listdir(input_folder) if path.isdir(path.join(input_folder, x)) and x in valid_folders][0:10]
26+
[queue.append(path.join(block, x)) for x in children]
27+
start_time = time.time()
28+
convert(block, input_root_path, output_root_path, max_intensity)
29+
# print '%s, %.1f sec/blocks\n' % (block, time.time() - start_time)
30+
31+
32+
def calc_statistics(x):
33+
m = np.mean(x)
34+
c = x - m
35+
c.shape = (-1)
36+
return m, np.dot(c, c) / (c.size - 1)
37+
38+
39+
def convert(max_intensity, input_root_path, output_root_path, name_of_log_file):
40+
output_path = path.join(output_root_path, 'default.tif')
41+
#if path.exists(output_path):
42+
# return
43+
44+
tiff0 = np.float32(io.imread(path.join(input_root_path, 'default.0.tif')))
45+
46+
diff8b = np.uint8(tiff0 / ((2**16)-1) * 255.0)
47+
#io.imsave(input_root_path + "/test.tif", diff8b)
48+
49+
pyglass.Raster(diff8b).Export(output_path)
50+
51+
#if name_of_log_file != "default":
52+
# with open(name_of_log_file, 'w') as w:
53+
# w.write('1000000')
54+
55+
#pyglass.CombineRasters(pyglass.Raster(tiff0), pyglass.Raster(tiff1)).Export(output_path)
56+
57+
58+
def process_octree(input_root_path,
59+
output_root_path,
60+
name_of_log_file = "default",
61+
max_intensity=default_max_intensity,
62+
max_queue_size=default_max_queue_size,
63+
num_worker_threads=default_num_worker_threads):
64+
65+
convert(max_intensity, input_root_path, output_root_path, name_of_log_file)
66+
67+
if __name__ == '__main__':
68+
if len(sys.argv) == 3:
69+
sys.argv.append("default")
70+
process_octree(sys.argv[1], sys.argv[2], sys.argv[3])

‎singleThreadedCacherMultiChannel.py

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#! python2
2+
import sys
3+
import time
4+
from os import path, listdir, makedirs
5+
import numpy as np
6+
from skimage import io
7+
import pyglass
8+
9+
default_max_intensity = 15000.0
10+
default_num_worker_threads = 12
11+
default_max_queue_size = 12
12+
13+
14+
def worker(queue, max_intensity, input_root_path, output_root_path):
15+
while len(queue) > 0:
16+
block = queue.pop(0)
17+
input_folder = path.join(input_root_path, block)
18+
output_folder = path.join(output_root_path, block)
19+
try:
20+
makedirs(output_folder)
21+
except OSError:
22+
if not path.isdir(output_folder):
23+
raise
24+
valid_folders = [str(n) for n in range(1, 9)]
25+
children = [x for x in listdir(input_folder) if path.isdir(path.join(input_folder, x)) and x in valid_folders][0:10]
26+
[queue.append(path.join(block, x)) for x in children]
27+
start_time = time.time()
28+
convert(block, input_root_path, output_root_path, max_intensity)
29+
print '%s, %.1f sec/blocks\n' % (block, time.time() - start_time)
30+
31+
32+
def calc_statistics(x):
33+
m = np.mean(x)
34+
c = x - m
35+
c.shape = (-1)
36+
return m, np.dot(c, c) / (c.size - 1)
37+
38+
39+
def convert(max_intensity, input_root_path, output_root_path, name_of_log_file):
40+
output_path = path.join(output_root_path, 'default.tif')
41+
if path.exists(output_path):
42+
return
43+
44+
tiff0 = np.float32(io.imread(path.join(input_root_path, 'default.0.tif')))
45+
tiff1 = np.float32(io.imread(path.join(input_root_path, 'default.1.tif')))
46+
47+
(m0, std0) = calc_statistics(tiff0)
48+
(m1, std1) = calc_statistics(tiff1)
49+
50+
std = (std0 + std1) / 2
51+
diff = tiff0 * std / std0 - tiff1 * std / std1 - (m0 / std0 - m1 / std1) * std
52+
diff *= (128.0 / max_intensity)
53+
diff8b = np.uint8(np.clip(diff, -128.0, 127.0) + 128.0)
54+
55+
pyglass.Raster(diff8b).Export(output_path)
56+
57+
if name_of_log_file != "default":
58+
with open(name_of_log_file, 'w') as w:
59+
w.write('1000000')
60+
61+
#pyglass.CombineRasters(pyglass.Raster(tiff0), pyglass.Raster(tiff1)).Export(output_path)
62+
63+
64+
def process_octree(input_root_path,
65+
output_root_path,
66+
name_of_log_file = "default",
67+
max_intensity=default_max_intensity,
68+
max_queue_size=default_max_queue_size,
69+
num_worker_threads=default_num_worker_threads):
70+
71+
convert(max_intensity, input_root_path, output_root_path, name_of_log_file)
72+
73+
if __name__ == '__main__':
74+
if len(sys.argv) == 3:
75+
sys.argv.append("default")
76+
process_octree(sys.argv[1], sys.argv[2], sys.argv[3])

‎syPI.py

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import requests
2+
import json
3+
import code
4+
5+
###############################################################################
6+
###############################################################################
7+
# Creates a project with an empty tiff file in the center of it
8+
# createEmptyProject('projectName', 'C:/syPI/')
9+
10+
def createEmptyProject(name, pathToSYGFolder):
11+
import tifffile
12+
import numpy as np
13+
import os
14+
15+
cwd = os.getcwd()
16+
empty = np.zeros((1,1,1))
17+
tifffile.imsave('empty.tif', np.uint8(empty))
18+
pathToFile = cwd + '\\empty.tif'
19+
20+
if '/' in pathToSYGFolder:
21+
pathToSYGFolder = pathToSYGFolder.replace('/','\\')
22+
23+
data = {'numChannels': 1, 'name': name, 'convertMax': '255.0', \
24+
'showFrameTemplateString': 0, 'convertMin': '0.0', \
25+
'dataEnum': 'UINT8', 'voxelZ': '1', 'voxelX': '1', \
26+
'voxelY': '1', 'showSliceTemplateString': 0, \
27+
'masterFileList': [[pathToFile]], \
28+
'pathText1': pathToFile, \
29+
'path': pathToSYGFolder, 'extractChannel': 0, 'overwrite': 0, \
30+
'convertType': 'NONE'}
31+
r = requests.post('http://127.0.0.1:8000/syglass/createProject', json.dumps(data))
32+
print(r.status_code, r.reason)
33+
34+
fullPath = pathToSYGFolder + name + '\\' + name + '.syg'
35+
36+
dataForLibrary = {'body': {'path': fullPath, \
37+
'type': 'NONE', 'modality': 'OTHER', 'voxelSize': '(1, 1, 1)',\
38+
'name': name}, 'name': name}
39+
40+
r = requests.post('http://127.0.0.1:8000/syglass/VolumeLibrary/putEntry', json.dumps(dataForLibrary))
41+
print(r.status_code, r.reason)
42+
43+
###############################################################################
44+
###############################################################################
45+
# Create a project with a single tiff stacks
46+
# Example
47+
# createProject('projectName', 'C:/syPI/empty.tif', 'C:/syPI/')
48+
49+
def createProject(name, pathToFile, pathToSYGFolder):
50+
if '/' in pathToFile:
51+
pathToFile = pathToFile.replace('/','\\')
52+
53+
if '/' in pathToSYGFolder:
54+
pathToSYGFolder = pathToSYGFolder.replace('/','\\')
55+
56+
data = {'numChannels': 1, 'name': name, 'convertMax': '255.0', \
57+
'showFrameTemplateString': 0, 'convertMin': '0.0', \
58+
'dataEnum': 'UINT8', 'voxelZ': '1', 'voxelX': '1', \
59+
'voxelY': '1', 'showSliceTemplateString': 0, \
60+
'masterFileList': [[pathToFile]], \
61+
'pathText1': pathToFile, \
62+
'path': pathToSYGFolder, 'extractChannel': 0, 'overwrite': 0, \
63+
'convertType': 'NONE'}
64+
r = requests.post('http://127.0.0.1:8000/syglass/createProject', json.dumps(data))
65+
print(r.status_code, r.reason)
66+
67+
fullPath = pathToSYGFolder + name + '\\' + name + '.syg'
68+
69+
dataForLibrary = {'body': {'path': fullPath, \
70+
'type': 'NONE', 'modality': 'OTHER', 'voxelSize': '(1, 1, 1)',\
71+
'name': name}, 'name': name}
72+
73+
r = requests.post('http://127.0.0.1:8000/syglass/VolumeLibrary/putEntry', json.dumps(dataForLibrary))
74+
print(r.status_code, r.reason)
75+
76+
###############################################################################
77+
###############################################################################
78+
# Add a list of OBJ files to the project
79+
# EXAMPLE:
80+
# import glob
81+
# l = glob.glob('C:/syPI/meshes/*.obj')
82+
# l = [i.replace('/','\\') for i in l]
83+
# print(l)
84+
# addMeshesToProject('C:/syPI/projectName/projectName.syg', 'default', l)
85+
86+
def addMeshesToProject(pathToProject, nameOfExperiment, listOfMeshes):
87+
#{'projectPath': 'C:\\syPI\\name4\\name4.syg', 'dir': 'default', 'path': 'C:\\Users\\mmore\\Downloads\\SevenSisters\\VCN_c09_Axon01.obj\r\nC:\\Users\\mmore\\Downloads\\SevenSisters\\VCN_c09_cellbody.obj\r\nC:\\Users\\mmore\\Downloads\\SevenSisters\\VCN_c09_Dendrite01.obj\r\nC:\\Users\\mmore\\Downloads\\SevenSisters\\VCN_c09_myelin01.obj\r\nC:\\Users\\mmore\\Downloads\\SevenSisters\\VCN_c09_nucleus01.obj\r\n'}
88+
89+
if '/' in pathToProject:
90+
pathToProject = pathToProject.replace('/','\\')
91+
data = {'projectPath': pathToProject, 'dir': nameOfExperiment, 'path': '\r\n'.join(listOfMeshes)}
92+
r = requests.post('http://127.0.0.1:8000/syglass/importMeshOBJs', json.dumps(data))
93+
print(r.status_code, r.reason)
94+
95+
###############################################################################
96+
###############################################################################
97+
# Add DVID data to a project
98+
# Example:
99+
# addDVIDToProject('emdata.janelia.org:80', '90823d3056a044608ebbb5740b6b46c1', 'grayscalejpeg', 'default', 'C:/syPI/projectName/projectName.syg')
100+
def addDVIDToProject(url, uuid, dvidLabelName, nameOfExperiment, pathToProject):
101+
if '/' in pathToProject:
102+
pathToProject = pathToProject.replace('/','\\')
103+
data = {'url': url, 'uuid': uuid, 'dataType': dvidLabelName, 'dir': nameOfExperiment, 'projectPath': pathToProject}
104+
r = requests.post('http://127.0.0.1:8000/syglass/setDVIDFields', json.dumps(data))
105+
print(r.status_code, r.reason)
106+
107+
###############################################################################
108+
###############################################################################
109+
#need to do three things
110+
# 1. read transform data file, convert
111+
# 2. add json file to folder with top level data
112+
# 3. append project path to VolumeLibrary
113+
114+
def buildMouseLightProject(path, name, pathToTransformationFile):
115+
# read transform data file
116+
transformData = dict(line.strip().split(':', 1) for line in open(pathToTransformationFile, 'r'))
117+
level = int(transformData['nl'])
118+
sx = float(transformData['sx']) / 1000.0 / 2**(level-1)
119+
sy = float(transformData['sy']) / 1000.0 / 2**(level-1)
120+
sz = float(transformData['sz']) / 1000.0 / 2**(level-1)
121+
ox = int(transformData['ox']) / 1000.0
122+
oy = int(transformData['oy']) / 1000.0
123+
oz = int(transformData['oz']) / 1000.0
124+
125+
# add json data to folder with top level data
126+
data = {'voxelsize_used_um': [sx, sy, sz], 'origin_um': [ox, oy, oz], 'levels': level-1, 'block_filename': "default.tif"}
127+
with open(path + name + '.json', 'w') as outfile:
128+
json.dump(data, outfile)
129+
130+
dataForLibrary = {'body': {'path': path + name + '.json', \
131+
'type': 'dense', 'modality': 'OTHER', 'voxelSize': '',\
132+
'name': name}, 'name': name}
133+
134+
r = requests.post('http://127.0.0.1:8000/syglass/VolumeLibrary/putEntry', json.dumps(dataForLibrary))
135+
print(r.status_code, r.reason)
136+
137+
138+
buildMouseLightProject('Y:/SAMPLES/2017-09-25/syglass-ch0/', '2017-09-25_ch0.json', 'Y:/SAMPLES/2017-09-25/transform.txt')

0 commit comments

Comments
 (0)