USEFULL
/
qw-gallery-scenery
mirror of https://github.com/qwertyforce/scenery.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

scenery is working again 

Renovated to support new version of ambience. Deleted some unused functionality (may bring it back later)
pull/8/head
qwertyforce 2022-08-30 22:22:35 +03:00
parent dfd6ffe02b
commit 24d0cd0734
66 changed files with 389 additions and 5174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
additional.d.ts vendored
View File

@ -3,7 +3,6 @@ declare global {
interface ProcessEnv {
api_domain: string,
recaptcha_site_key: string,
reverse_search_url: string,
domain: string,
ipns: string
}

85
ambience/bulk_calc.py
View File

@ -1,85 +0,0 @@
from os import path,listdir,remove
import subprocess
import imagesize
from tqdm import tqdm
AKAZE_MANUAL_DEDUP=False
dir_path = path.dirname(path.realpath(__file__))
print(dir_path)
def clean_up():
print("=========CLEANING UP=========")
if path.exists("./import_file_name_to_scenery_img_data.txt"): remove("./import_file_name_to_scenery_img_data.txt")
if path.exists("./modules/phash/trained_import.index"): remove("./modules/phash/trained_import.index")
if path.exists("./modules/phash/import_phashes.db"): remove("./modules/phash/import_phashes.db")
if path.exists("./modules/akaze/import_akaze.db"): remove("./modules/akaze/import_akaze.db")
if path.exists("./modules/akaze/trained_import.index"): remove("./modules/akaze/trained_import.index")
clean_up()
print("=========DELETING LOW-RES IMAGES=========")
IMAGE_PATH="./../import/images"
file_names=listdir(IMAGE_PATH)
for file_name in tqdm(file_names):
img_path=IMAGE_PATH+"/"+file_name
width, height = imagesize.get(img_path)
if(width*height<1024*1024):
print(f'deleted {file_name}')
remove(img_path)
print("=========AUTOROTATE JPEG (EXIF)=========")
subprocess.call("exiftran -ai *.jpg",shell=True,cwd="../import/images/")
subprocess.call("exiftran -ai *.jpeg",shell=True,cwd="../import/images/")
print("=========DEDUP SHA256=========")
subprocess.call(["node","dedupe_import_sha256.js"],shell=True,cwd="../dist/server/bulk_import_images/")
print("=========GENERATING IMPORT PHASHES=========")
subprocess.call(["python3","generate_import_phashes.py"],shell=True,cwd="./modules/phash/")
print("=========TRAIN IVF FOR IMPORT=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","train_ivf_import.py"],shell=True,cwd="./modules/phash/")
print("=========PHASH DEDUP INTERNAL=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","phash_dedup_internal.py"],shell=True,cwd="./modules/phash/")
print("=========PHASH DEDUP=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","phash_dedup.py"],shell=True,cwd="./modules/phash/")
print("=========GENERATING AKAZE DESCRIPTORS=========")
subprocess.call(["python3","akaze_bulk_calculate_import.py"],shell=True,cwd="./modules/akaze/")
print("=========AKAZE TRAIN_IVF_import=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","train_ivf_import.py"],shell=True,cwd="./modules/akaze/")
if AKAZE_MANUAL_DEDUP:
print("=========AKAZE DEDUP=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","akaze_dedup.py"],shell=True,cwd="./modules/akaze/")
print("press enter to conitnue")
input()
else:
print("=========SKIPPING AKAZE DEDUP=========")
print("=========IMPORTING IMAGES INTO SCENERY=========")
subprocess.call(["node","bulk_import_images_without_check.js"],shell=True,cwd="../dist/server/bulk_import_images/")
print("=========AUTO TAGGING IMAGES=========")
subprocess.call(["python3","resnet_image_tagging.py"],shell=True,cwd="./modules/nn/")
print("=========IMPORTING TAGS INTO SCENERY=========")
subprocess.call(["node","bulk_import_tags.js"],shell=True,cwd="../dist/server/bulk_import_images/")
print("=========PHASH IMPORT INTO MAIN DB=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","phash_import.py"],shell=True,cwd="./modules/phash/")
print("=========PHASH TRAIN_IVF=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","train_ivf.py"],shell=True,cwd="./modules/phash/")
print("=========AKAZE IMPORT INTO MAIN DB=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","akaze_import.py"],shell=True,cwd="./modules/akaze/")
print("=========AKAZE TRAIN_IVF=========")
subprocess.call(["conda", "activate", "my_new_environment","&&","python","train_ivf.py"],shell=True,cwd="./modules/akaze/")
print("=========GENERATING NN FEATURES=========")
subprocess.call(["python3","clip_generate_vectors.py"],shell=True,cwd="./modules/nn/")
print("=========GENERATING RGB HISTOGRAMS=========")
subprocess.call(["python3","generate_rgb_histograms.py"],shell=True,cwd="./modules/histogram/")
clean_up()

7
ambience/config/config.js
View File

@ -1,7 +0,0 @@
export default {
server_port:"44444",
akaze_microservice_url:"http://localhost:33333",
nn_microservice_url:"http://localhost:33334",
hist_microservice_url:"http://localhost:33335",
phash_microservice_url:"http://localhost:33336",
}

43
ambience/microservices.config.js
View File

@ -1,43 +0,0 @@
module.exports = {
"apps": [
{
name: 'ambience_microservice',
script: 'index.js',
log_file: 'ambience_microservice.txt',
time: true,
cwd: "./ambience/dist/server"
},
{
name: 'akaze_microservice',
script: 'akaze_web.py',
log_file: 'akaze_microservice.txt',
interpreter: "python3",
time: true,
cwd: "./ambience/modules/akaze"
},
{
name: 'nn_microservice',
script: 'nn_web.py',
log_file: 'nn_microservice.txt',
interpreter: "python3",
time: true,
cwd: "./ambience/modules/nn"
},
{
name: 'phash_microservice',
script: 'phash_web.py',
log_file: 'phash_microservice.txt',
interpreter: "python3",
time: true,
cwd: "./ambience/modules/phash"
},
{
name: 'hist_microservice',
script: 'rgb_histogram_web.py',
log_file: 'histogram_microservice.txt',
interpreter: "python3",
time: true,
cwd: "./ambience/modules/histogram"
}
],
};

182
ambience/modules/akaze/akaze_bulk_calculate_import.py
View File

@ -1,182 +0,0 @@
import cv2
import numpy as np
from os import listdir
import math
from joblib import Parallel, delayed
from numba import jit
from numba.core import types
from numba.typed import Dict
import sqlite3
import io
conn = sqlite3.connect('import_akaze.db')
IMAGE_PATH = "./../../../import/images"
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS akaze(
id TEXT NOT NULL UNIQUE PRIMARY KEY,
akaze_features BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM akaze WHERE id=(?))'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM akaze WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM akaze'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id, akaze_features):
cursor = conn.cursor()
query = '''INSERT INTO akaze(id, akaze_features) VALUES (?,?)'''
cursor.execute(query, (id, akaze_features))
conn.commit()
def resize_img_to_threshold(img):
height, width = img.shape
threshold = 3000*3000
if height*width > threshold:
k = math.sqrt(height*width/(threshold))
img = cv2.resize(img, (round(width/k), round(height/k)), interpolation=cv2.INTER_LINEAR)
return img
@jit(nopython=True, cache=True, fastmath=True)
def check_distance(keypoint_x, keypoint_y, keypoints, keypoints_neighbors):
skip_flag = False
for keyp in keypoints:
if keyp[0] == 0 and keyp[1] == 0: #_keypoints is zeroed
break
dist = math.sqrt((keypoint_x-keyp[0])**2 + (keypoint_y-keyp[1])**2)
if dist < 40:
pseudohash = keyp[0]+keyp[1]
if not pseudohash in keypoints_neighbors:
keypoints_neighbors[pseudohash] = 1
if keypoints_neighbors[pseudohash] >= 3:
skip_flag = True
continue
else:
keypoints_neighbors[pseudohash] += 1
return skip_flag
def calculate_descr(img):
AKAZE = cv2.AKAZE_create(threshold=0) # can't serialize, hence init is here
img = resize_img_to_threshold(img)
height = img.shape[0]
width = img.shape[1]
height_divided_by_2 = height//2
width_divided_by_2 = width//2
kps = AKAZE.detect(img, None)
kps = sorted(kps, key=lambda x: x.response, reverse=True)
descriptors_count = [0, 0, 0, 0]
keypoints = []
_keypoints = np.zeros((256, 2))
keypoints_neighbors = Dict.empty(key_type=types.float64, value_type=types.int64)
for keypoint in kps:
keypoint_x, keypoint_y = keypoint.pt
if len(keypoints) != 0:
skip_keypoint = check_distance(keypoint_x, keypoint_y, _keypoints, keypoints_neighbors)
if skip_keypoint:
continue
if sum(descriptors_count) == 64*4:
break
if descriptors_count[0] < 64 and 0 < keypoint_y < height_divided_by_2 and 0 < keypoint_x < width_divided_by_2:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[0] += 1
continue
if descriptors_count[1] < 64 and 0 < keypoint_y < height_divided_by_2 and width_divided_by_2 < keypoint_x < width:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[1] += 1
continue
if descriptors_count[2] < 64 and height_divided_by_2 < keypoint_y < height and 0 < keypoint_x < width_divided_by_2:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[2] += 1
continue
if descriptors_count[3] < 64 and height_divided_by_2 < keypoint_y < height and 0 < width_divided_by_2 < keypoint_x < width:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[3] += 1
continue
_, desc1 = AKAZE.compute(img, keypoints)
return desc1
def sync_db():
ids_in_db = set(get_all_ids())
for file_name in file_names:
if file_name in ids_in_db:
ids_in_db.remove(file_name)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
def calc_features(file_name):
img_path = IMAGE_PATH+"/"+file_name
query_image = cv2.imread(img_path, 0)
if query_image is None:
return None
descs = calculate_descr(query_image)
if descs is None:
return None
descs_bin = adapt_array(descs)
return (file_name, descs_bin)
file_names = listdir(IMAGE_PATH)
create_table()
sync_db()
new_images = []
for file_name in file_names:
if check_if_exists_by_id(file_name):
continue
new_images.append(file_name)
new_images = [new_images[i:i + 5000] for i in range(0, len(new_images), 5000)]
for batch in new_images:
descriptors = Parallel(n_jobs=-1, verbose=1)(delayed(calc_features)(file_name) for file_name in batch)
descriptors = [i for i in descriptors if i] # remove None's
print("pushing data to db")
conn.executemany('''INSERT INTO akaze(id, akaze_features) VALUES (?,?)''', descriptors)
conn.commit()

174
ambience/modules/akaze/akaze_bulk_calculate_public.py
View File

@ -1,174 +0,0 @@
import cv2
import numpy as np
from os import listdir
import math
from numba import jit
from numba.core import types
from numba.typed import Dict
from joblib import Parallel, delayed
import sqlite3
import io
conn = sqlite3.connect('akaze.db')
IMAGE_PATH="./../../../public/images"
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS akaze(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
akaze_features BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM akaze WHERE id=(?))'''
cursor.execute(query,(id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM akaze WHERE id=(?)'''
cursor.execute(query,(id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM akaze'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:el[0],all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id,akaze_features):
cursor = conn.cursor()
query = '''INSERT INTO akaze(id, akaze_features) VALUES (?,?)'''
cursor.execute(query,(id,akaze_features))
conn.commit()
def resize_img_to_threshold(img):
height, width = img.shape
threshold = 3000*3000
if height*width > threshold:
k = math.sqrt(height*width/(threshold))
img = cv2.resize(img, (round(width/k), round(height/k)), interpolation=cv2.INTER_LINEAR)
return img
@jit(nopython=True, cache=True, fastmath=True)
def check_distance(keypoint_x, keypoint_y, keypoints, keypoints_neighbors):
skip_flag = False
for keyp in keypoints:
if keyp[0] == 0 and keyp[1] == 0: #_keypoints is zeroed
break
dist = math.sqrt((keypoint_x-keyp[0])**2 + (keypoint_y-keyp[1])**2)
if dist < 40:
pseudohash = keyp[0]+keyp[1]
if not pseudohash in keypoints_neighbors:
keypoints_neighbors[pseudohash] = 1
if keypoints_neighbors[pseudohash] >= 3:
skip_flag = True
continue
else:
keypoints_neighbors[pseudohash] += 1
return skip_flag
def calculate_descr(img):
AKAZE = cv2.AKAZE_create(threshold=0) # can't serialize, hence init is here
img = resize_img_to_threshold(img)
height = img.shape[0]
width = img.shape[1]
height_divided_by_2 = height//2
width_divided_by_2 = width//2
kps = AKAZE.detect(img, None)
kps = sorted(kps, key=lambda x: x.response, reverse=True)
descriptors_count = [0, 0, 0, 0]
keypoints = []
_keypoints = np.zeros((256, 2))
keypoints_neighbors = Dict.empty(key_type=types.float64, value_type=types.int64)
for keypoint in kps:
keypoint_x, keypoint_y = keypoint.pt
if len(keypoints) != 0:
skip_keypoint = check_distance(keypoint_x, keypoint_y, _keypoints, keypoints_neighbors)
if skip_keypoint:
continue
if sum(descriptors_count) == 64*4:
break
if descriptors_count[0] < 64 and 0 < keypoint_y < height_divided_by_2 and 0 < keypoint_x < width_divided_by_2:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[0] += 1
continue
if descriptors_count[1] < 64 and 0 < keypoint_y < height_divided_by_2 and width_divided_by_2 < keypoint_x < width:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[1] += 1
continue
if descriptors_count[2] < 64 and height_divided_by_2 < keypoint_y < height and 0 < keypoint_x < width_divided_by_2:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[2] += 1
continue
if descriptors_count[3] < 64 and height_divided_by_2 < keypoint_y < height and 0 < width_divided_by_2 < keypoint_x < width:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[3] += 1
continue
_, desc1 = AKAZE.compute(img, keypoints)
return desc1
def sync_db():
ids_in_db=set(get_all_ids())
for file_name in file_names:
file_id=int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) #Fix this
print(f"deleting {id}")
def calc_features(file_name):
file_id=int(file_name[:file_name.index('.')])
img_path=IMAGE_PATH+"/"+file_name
query_image=cv2.imread(img_path,0)
if query_image is None:
return None
descs=calculate_descr(query_image)
descs_bin=adapt_array(descs)
print(file_name)
return (file_id,descs_bin)
file_names=listdir(IMAGE_PATH)
create_table()
sync_db()
new_images=[]
for file_name in file_names:
file_id=int(file_name[:file_name.index('.')])
if check_if_exists_by_id(file_id):
continue
new_images.append(file_name)
new_images=[new_images[i:i + 5000] for i in range(0, len(new_images), 5000)]
for batch in new_images:
descriptors=Parallel(n_jobs=-1,verbose=1)(delayed(calc_features)(file_name) for file_name in batch)
descriptors= [i for i in descriptors if i] #remove None's
print("pushing data to db")
conn.executemany('''INSERT INTO akaze(id, akaze_features) VALUES (?,?)''', descriptors)
conn.commit()

120
ambience/modules/akaze/akaze_dedup.py
View File

@ -1,120 +0,0 @@
import faiss
import numpy as np
import sqlite3
import io
from tqdm import tqdm
conn_import = sqlite3.connect('import_akaze.db')
index = None
POINT_ID = 0
point_id_to_image_id_map = {}
image_id_to_point_ids_map = {}
def import_get_all_data():
cursor = conn_import.cursor()
query = '''
SELECT id, akaze_features
FROM akaze
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: (el[0], convert_array(el[1])), all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def median(lst):
list_length = len(lst)
index = (list_length - 1) // 2
if (list_length % 2):
return lst[index][1]
else:
return (lst[index][1] + lst[index + 1][1])/2.0
def _akaze_reverse_search(D, I, file_name):
levels_threshold = [2, 4, 6, 10]
levels = [{}, {}, {}, {}]
all_points = {}
for i in range(len(I)):
point_dist = D[i]
if(point_dist > 65):
continue
point_id = I[i]
image_id = point_id_to_image_id_map[point_id]
if image_id == file_name: # skip original image
continue
if point_dist <= 5:
levels[0][image_id] = levels[0].get(image_id, 0)+1
if point_dist <= 10:
levels[1][image_id] = levels[1].get(image_id, 0)+1
if point_dist <= 15:
levels[2][image_id] = levels[2].get(image_id, 0)+1
if point_dist <= 32:
levels[3][image_id] = levels[3].get(image_id, 0)+1
all_points[image_id] = all_points.get(image_id, 0)+1
for i in range(4):
if(len(levels[i]) > 0):
sorted_levels = sorted(levels[i].items(), key=lambda item: item[1])
if sorted_levels[-1][1] >= levels_threshold[i]:
print({"data": sorted_levels[-1], "level": i})
return [sorted_levels[-1][0]]
if len(all_points) >= 2:
sorted_all_points = sorted(all_points.items(), key=lambda item: item[1])
median_number_of_points = median(sorted_all_points)
if sorted_all_points[-1][1] >= 10:
if((sorted_all_points[-1][1]/median_number_of_points) >= 3):
print(sorted_all_points[-1])
return [sorted_all_points[-1][0]]
return []
def akaze_reverse_search(target_features, file_name):
D, I = index.search(target_features, 2)
res = _akaze_reverse_search(D.reshape(-1), I.reshape(-1), file_name)
return res
def dedup():
global index, POINT_ID
try:
index = faiss.read_index_binary("trained_import.index")
except:
print("trained_import.index not found. exit()")
exit()
all_data = import_get_all_data()
for obj in tqdm(all_data):
image_id = obj[0]
features = obj[1]
point_ids = np.arange(
start=POINT_ID, stop=POINT_ID+len(features), dtype=np.int64)
for point_id in point_ids:
point_id_to_image_id_map[point_id] = image_id
image_id_to_point_ids_map[image_id] = point_ids
POINT_ID += len(features)
index.add_with_ids(features, point_ids)
print("Index is ready")
deleted = []
for x in tqdm(all_data):
filename = x[0]
if filename in deleted:
continue
features = x[1]
res = akaze_reverse_search(features, filename)
if len(res) > 0 and res[0] != filename:
print("=============")
print(filename)
print(res[0])
print("=============")
deleted.extend(res)
dedup()

46
ambience/modules/akaze/akaze_import.py
View File

@ -1,46 +0,0 @@
import json
import sqlite3
conn_import = sqlite3.connect('import_akaze.db')
conn = sqlite3.connect('akaze.db')
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS akaze(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
akaze_features BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def get_all_data():
cursor = conn_import.cursor()
query = '''
SELECT id, akaze_features
FROM akaze
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:(el[0],el[1]),all_rows))
create_table()
try:
with open('./../../import_file_name_to_scenery_img_data.txt', encoding="utf-8") as json_file:
filename_to_img_id_map = json.load(json_file)
except:
print("import_file_name_to_scenery_img_data.txt not found")
exit()
akaze_data_import=get_all_data()
img_id_akaze=[]
for akaze_data in akaze_data_import:
filename=akaze_data[0]
akaze=akaze_data[1]
if filename in filename_to_img_id_map:
img_id=filename_to_img_id_map[filename]["image_id"]
img_id_akaze.append((img_id,akaze))
conn.executemany('''INSERT INTO akaze(id, akaze_features) VALUES (?,?)''', img_id_akaze)
conn.commit()

341
ambience/modules/akaze/akaze_web.py
View File

@ -1,341 +0,0 @@
import uvicorn
if __name__ == '__main__':
uvicorn.run('akaze_web:app', host='127.0.0.1', port=33333, log_level="info")
import faiss
import cv2
from os import listdir
import numpy as np
import math
from numba import jit
from numba.core import types
from numba.typed import Dict
from tqdm import tqdm
from fastapi import FastAPI, File, Form, HTTPException, Response, status
from pydantic import BaseModel
import sqlite3
import io
conn = sqlite3.connect('akaze.db')
AKAZE = cv2.AKAZE_create(threshold=0)
index = None
point_id_to_image_id_map = {}
image_id_to_point_ids_map = {}
IMAGE_PATH = "./../../../public/images"
POINT_ID = 0
FIND_MIRRORED = True
def init_index():
global index, POINT_ID
try:
index = faiss.read_index_binary("trained.index")
except: #temporary index
d = 61*8
quantizer = faiss.IndexBinaryFlat(d)
index = faiss.IndexBinaryIVF(quantizer, d, 1)
index.nprobe = 1
index.train(np.array([np.zeros(61)], dtype=np.uint8))
all_ids = get_all_ids()
for image_id in tqdm(all_ids):
features = convert_array(get_akaze_features_by_id(image_id))
point_ids = np.arange(start=POINT_ID, stop=POINT_ID+len(features), dtype=np.int64)
for point_id in point_ids:
point_id_to_image_id_map[point_id] = image_id
image_id_to_point_ids_map[image_id] = point_ids
POINT_ID += len(features)
index.add_with_ids(features, point_ids)
print("Index is ready")
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS akaze(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
akaze_features BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def sync_db():
ids_in_db = set(get_all_ids())
file_names = listdir(IMAGE_PATH)
for file_name in file_names:
file_id = int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
print("db synced")
def add_descriptor(id, akaze_features):
cursor = conn.cursor()
query = '''INSERT INTO akaze(id, akaze_features )VALUES (?,?)'''
cursor.execute(query, (id, akaze_features))
conn.commit()
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM akaze WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM akaze'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def get_akaze_features_by_id(id):
cursor = conn.cursor()
query = '''
SELECT akaze_features
FROM akaze
WHERE id = (?)
'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0]
def add_descriptor(id, akaze_features):
cursor = conn.cursor()
query = '''INSERT INTO akaze(id, akaze_features) VALUES (?,?)'''
cursor.execute(query, (id, akaze_features))
conn.commit()
def read_img_file(image_data):
return np.fromstring(image_data, np.uint8)
def resize_img_to_threshold(img):
height, width = img.shape
threshold = 3000*3000
if height*width > threshold:
k = math.sqrt(height*width/(threshold))
img = cv2.resize(img, (round(width/k), round(height/k)), interpolation=cv2.INTER_LINEAR)
return img
def preprocess_image(image_buffer):
img = cv2.imdecode(read_img_file(image_buffer), 0)
img = resize_img_to_threshold(img)
return img
@jit(nopython=True, cache=True, fastmath=True)
def check_distance(keypoint_x, keypoint_y, keypoints, keypoints_neighbors):
skip_flag = False
for keyp in keypoints:
if keyp[0] == 0 and keyp[1] == 0: # _keypoints is zeroed
break
dist = math.sqrt((keypoint_x-keyp[0])**2 + (keypoint_y-keyp[1])**2)
if dist < 40:
pseudohash = keyp[0]+keyp[1]
if not pseudohash in keypoints_neighbors:
keypoints_neighbors[pseudohash] = 1
if keypoints_neighbors[pseudohash] >= 3:
skip_flag = True
continue
else:
keypoints_neighbors[pseudohash] += 1
return skip_flag
def calculate_descr(img, mirrored=False):
if mirrored:
img = cv2.flip(img, 1)
height = img.shape[0]
width = img.shape[1]
height_divided_by_2 = height//2
width_divided_by_2 = width//2
kps = AKAZE.detect(img, None)
if kps is None:
return None
kps = sorted(kps, key=lambda x: x.response, reverse=True)
descriptors_count = [0, 0, 0, 0]
keypoints = []
_keypoints = np.zeros((256, 2))
keypoints_neighbors = Dict.empty(key_type=types.float64, value_type=types.int64)
for keypoint in kps:
keypoint_x, keypoint_y = keypoint.pt
if len(keypoints) != 0:
skip_keypoint = check_distance(keypoint_x, keypoint_y, _keypoints, keypoints_neighbors)
if skip_keypoint:
continue
if sum(descriptors_count) == 64*4:
break
if descriptors_count[0] < 64 and 0 < keypoint_y < height_divided_by_2 and 0 < keypoint_x < width_divided_by_2:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[0] += 1
continue
if descriptors_count[1] < 64 and 0 < keypoint_y < height_divided_by_2 and width_divided_by_2 < keypoint_x < width:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[1] += 1
continue
if descriptors_count[2] < 64 and height_divided_by_2 < keypoint_y < height and 0 < keypoint_x < width_divided_by_2:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[2] += 1
continue
if descriptors_count[3] < 64 and height_divided_by_2 < keypoint_y < height and 0 < width_divided_by_2 < keypoint_x < width:
keypoints.append(keypoint)
_keypoints[len(keypoints)-1][0] = keypoint.pt[0]
_keypoints[len(keypoints)-1][1] = keypoint.pt[1]
descriptors_count[3] += 1
continue
_, desc1 = AKAZE.compute(img, keypoints)
return desc1
"""
akaze_reverse_search
level - hamming distance range - points to be considered as a match
level 0 - <=5 - 2
level 1 - <=10 - 4
level 2 - <=15 - 6
level 3 - <=32 - 10
"""
def median(lst):
list_length = len(lst)
index = (list_length - 1) // 2
if (list_length % 2):
return lst[index][1]
else:
return (lst[index][1] + lst[index + 1][1])/2.0
def akaze_reverse_search(img, mirrored=False):
levels_threshold = [2, 4, 6, 10]
levels = [{}, {}, {}, {}]
all_points = {}
descs = calculate_descr(img, mirrored)
if descs is None:
return []
D, I = index.search(descs, 1)
for i in range(len(I)):
point_dist = D[i][0]
if(point_dist > 65):
continue
point_id = I[i][0]
image_id = point_id_to_image_id_map[point_id]
if point_dist <= 5:
levels[0][image_id] = levels[0].get(image_id, 0)+1
if point_dist <= 10:
levels[1][image_id] = levels[1].get(image_id, 0)+1
if point_dist <= 15:
levels[2][image_id] = levels[2].get(image_id, 0)+1
if point_dist <= 32:
levels[3][image_id] = levels[3].get(image_id, 0)+1
all_points[image_id] = all_points.get(image_id, 0)+1
print(levels)
for i in range(4):
if(len(levels[i]) > 0):
sorted_levels = sorted(levels[i].items(), key=lambda item: item[1])
if sorted_levels[-1][1] >= levels_threshold[i]:
print({"data": sorted_levels[-1], "level": i})
return [sorted_levels[-1][0]]
if len(all_points) >= 2:
sorted_all_points = sorted(all_points.items(), key=lambda item: item[1])
median_number_of_points = median(sorted_all_points)
if sorted_all_points[-1][1] >= 10:
if((sorted_all_points[-1][1]/median_number_of_points) >= 3):
print(sorted_all_points[-1])
return [sorted_all_points[-1][0]]
return []
app = FastAPI()
@app.get("/")
async def read_root():
return {"Hello": "World"}
@app.post("/akaze_reverse_search")
async def akaze_reverse_search_handler(image: bytes = File(...)):
img = preprocess_image(image)
found_image = akaze_reverse_search(img)
if len(found_image) == 0 and FIND_MIRRORED:
found_image = akaze_reverse_search(img, mirrored=True)
return found_image
@app.post("/calculate_akaze_features")
async def calculate_akaze_features_handler(image: bytes = File(...), image_id: str = Form(...)):
global POINT_ID
image_id = int(image_id)
img = preprocess_image(image)
descs = calculate_descr(img)
if descs is None:
raise HTTPException(status_code=500, detail="No descritors for this image")
add_descriptor(image_id, adapt_array(descs))
point_ids = np.arange(start=POINT_ID, stop=POINT_ID + len(descs), dtype=np.int64)
POINT_ID += len(descs)
for point_id in point_ids:
point_id_to_image_id_map[point_id] = image_id
image_id_to_point_ids_map[image_id] = point_ids
index.add_with_ids(descs, point_ids)
return Response(status_code=status.HTTP_200_OK)
class Item(BaseModel):
image_id: str
@app.post("/delete_akaze_features")
async def delete_akaze_features_handler(item: Item):
image_id = int(item.image_id)
delete_descriptor_by_id(image_id)
if image_id in image_id_to_point_ids_map:
point_ids = image_id_to_point_ids_map[image_id]
index.remove_ids(point_ids)
for point_id in point_ids:
del point_id_to_image_id_map[point_id]
del image_id_to_point_ids_map[image_id]
return Response(status_code=status.HTTP_200_OK)
else:
raise HTTPException(status_code=500, detail="Image with this id is not found")
print(__name__)
if __name__ == 'akaze_web':
create_table()
sync_db()
init_index()

50
ambience/modules/akaze/train_ivf.py
View File

@ -1,50 +0,0 @@
import sqlite3
import numpy as np
import io
import faiss
from math import sqrt
conn = sqlite3.connect('akaze.db')
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM akaze'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:el[0],all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def get_akaze_features_by_id(id):
cursor = conn.cursor()
query = '''
SELECT akaze_features
FROM akaze
WHERE id = (?)
'''
cursor.execute(query,(id,))
all_rows = cursor.fetchone()
return all_rows[0]
def train():
all_descriptors=[]
all_ids=get_all_ids()
if len(all_ids)==0:
print("No images. exit()")
exit()
for id in all_ids:
x=convert_array(get_akaze_features_by_id(id))
all_descriptors.append(x)
all_descriptors=np.concatenate(all_descriptors, axis=0)
d=61*8
centroids = round(sqrt(all_descriptors.shape[0]))
print(f'centroids: {centroids}')
quantizer = faiss.IndexBinaryFlat(d)
index = faiss.IndexBinaryIVF(quantizer, d, centroids)
index.nprobe = 8
index.train(all_descriptors)
faiss.write_index_binary(index, "./" + "trained.index")
train()

41
ambience/modules/akaze/train_ivf_import.py
View File

@ -1,41 +0,0 @@
import sqlite3
import numpy as np
import io
import faiss
from math import sqrt
conn_import = sqlite3.connect('import_akaze.db')
def import_get_all_data():
cursor = conn_import.cursor()
query = '''
SELECT id, akaze_features
FROM akaze
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:(el[0],convert_array(el[1])),all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def train():
all_descriptors=[]
all_data=import_get_all_data()
if len(all_data)==0:
print("No images. exit()")
exit()
for x in all_data:
all_descriptors.append(x[1])
all_descriptors=np.concatenate(all_descriptors, axis=0)
d=61*8
centroids = round(sqrt(all_descriptors.shape[0]))
print(f'centroids: {centroids}')
quantizer = faiss.IndexBinaryFlat(d)
index = faiss.IndexBinaryIVF(quantizer, d, centroids)
index.nprobe = 8
index.train(all_descriptors)
faiss.write_index_binary(index, "./" + "trained_import.index")
train()

113
ambience/modules/histogram/generate_rgb_histograms.py
View File

@ -1,113 +0,0 @@
import cv2
from os import listdir
import numpy as np
from joblib import Parallel, delayed
import sqlite3
import io
conn = sqlite3.connect('rgb_histograms.db')
IMAGE_PATH = "./../../../public/images"
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS rgb_hists(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
rgb_histogram BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM rgb_hists WHERE id=(?))'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM rgb_hists WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM rgb_hists'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id, rgb_histogram):
cursor = conn.cursor()
query = '''INSERT INTO rgb_hists(id, rgb_histogram) VALUES (?,?)'''
cursor.execute(query, (id, rgb_histogram))
conn.commit()
def sync_db():
file_names = listdir(IMAGE_PATH)
ids_in_db = set(get_all_ids())
for file_name in file_names:
file_id = int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
def get_features(query_image):
query_hist_combined = cv2.calcHist([query_image], [0, 1, 2], None, [16, 16, 16], [0, 256, 0, 256, 0, 256])
query_hist_combined = query_hist_combined.flatten()
query_hist_combined = query_hist_combined*10000000
query_hist_combined = np.divide(query_hist_combined, query_image.shape[0]*query_image.shape[1], dtype=np.float32)
return query_hist_combined
def calc_hist(file_name):
file_id = int(file_name[:file_name.index('.')])
img_path = IMAGE_PATH+"/"+file_name
query_image = cv2.imread(img_path)
if query_image is None:
print(f'error reading {img_path}')
return None
if query_image.shape[2] == 1:
query_image = cv2.cvtColor(query_image, cv2.COLOR_GRAY2RGB)
else:
query_image = cv2.cvtColor(query_image, cv2.COLOR_BGR2RGB)
image_features = get_features(query_image)
image_features_bin = adapt_array(image_features)
return (file_id, image_features_bin)
file_names = listdir(IMAGE_PATH)
create_table()
sync_db()
new_images = []
for file_name in file_names:
file_id = int(file_name[:file_name.index('.')])
if check_if_exists_by_id(file_id):
continue
new_images.append(file_name)
new_images = [new_images[i:i + 5000] for i in range(0, len(new_images), 5000)]
for batch in new_images:
hists = Parallel(n_jobs=-1,verbose=1)(delayed(calc_hist)(file_name) for file_name in batch)
hists = [i for i in hists if i] # remove None's
print("pushing data to db")
conn.executemany('''INSERT INTO rgb_hists(id, rgb_histogram) VALUES (?,?)''', hists)
conn.commit()

179
ambience/modules/histogram/rgb_histogram_web.py
View File

@ -1,179 +0,0 @@
import uvicorn
if __name__ == '__main__':
uvicorn.run('rgb_histogram_web:app', host='127.0.0.1', port=33335, log_level="info")
from pydantic import BaseModel
from fastapi import FastAPI, File, Form, HTTPException, Response, status
import faiss
from os import listdir
import numpy as np
from tqdm import tqdm
import cv2
import sqlite3
import io
conn = sqlite3.connect('rgb_histograms.db')
IMAGE_PATH = "./../../../public/images"
sub_index = faiss.IndexFlat(4096, faiss.METRIC_L1)
index_id_map = faiss.IndexIDMap2(sub_index)
def init_index():
global index_flat
all_ids = get_all_ids()
for image_id in tqdm(all_ids):
features = convert_array(get_rgb_histogram_by_id(image_id))
index_id_map.add_with_ids(np.array([features]), np.int64([image_id]))
print("Index is ready")
def read_img_file(image_data):
return np.fromstring(image_data, np.uint8)
def get_rgb_histogram_by_id(id):
cursor = conn.cursor()
query = '''
SELECT rgb_histogram
FROM rgb_hists
WHERE id = (?)
'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0]
def get_features(image_buffer):
query_image = cv2.cvtColor(cv2.imdecode(read_img_file(image_buffer), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
query_hist_combined = cv2.calcHist([query_image], [0, 1, 2], None, [16, 16, 16], [0, 256, 0, 256, 0, 256])
query_hist_combined = query_hist_combined.flatten()
query_hist_combined = query_hist_combined*10000000
query_hist_combined = np.divide(query_hist_combined, query_image.shape[0]*query_image.shape[1], dtype=np.float32)
return query_hist_combined
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS rgb_hists(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
rgb_histogram BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM rgb_hists WHERE id=(?))'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM rgb_hists WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM rgb_hists'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id, rgb_histogram):
cursor = conn.cursor()
query = '''INSERT INTO rgb_hists(id, rgb_histogram) VALUES (?,?)'''
cursor.execute(query, (id, rgb_histogram))
conn.commit()
def sync_db():
ids_in_db = set(get_all_ids())
file_names = listdir(IMAGE_PATH)
for file_name in file_names:
file_id = int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
print("db synced")
def hist_similarity_search(target_features, n):
D, I = index_id_map.search(np.array([target_features]), n)
print(D, I)
similar = []
for img_id in I[0]:
similar.append(int(img_id))
return similar
app = FastAPI()
@app.get("/")
async def read_root():
return {"Hello": "World"}
@app.post("/calculate_hist_features")
async def calculate_hist_features_handler(image: bytes = File(...), image_id: str = Form(...)):
features = get_features(image)
add_descriptor(int(image_id), adapt_array(features))
index_id_map.add_with_ids(np.array([features]), np.int64([image_id]))
return Response(status_code=status.HTTP_200_OK)
class Item_image_id(BaseModel):
image_id: int
@app.post("/hist_get_similar_images_by_id")
async def get_similar_images_by_id_handler(item: Item_image_id):
try:
target_features = index_id_map.reconstruct(item.image_id)
similar = hist_similarity_search(target_features, 20)
return similar
except:
raise HTTPException(
status_code=500, detail="Image with this id is not found")
@app.post("/hist_get_similar_images_by_image_buffer")
async def hist_get_similar_images_by_image_buffer_handler(image: bytes = File(...)):
target_features = get_features(image)
similar = hist_similarity_search(target_features, 20)
return similar
@app.post("/delete_hist_features")
async def delete_hist_features_handler(item: Item_image_id):
delete_descriptor_by_id(item.image_id)
index_id_map.remove_ids(np.int64([item.image_id]))
return Response(status_code=status.HTTP_200_OK)
print(__name__)
if __name__ == 'rgb_histogram_web':
create_table()
sync_db()
init_index()

108
ambience/modules/nn/clip_generate_vectors.py
View File

@ -1,108 +0,0 @@
import torch
import clip
from os import listdir
import numpy as np
from PIL import Image
import sqlite3
import io
from tqdm import tqdm
conn = sqlite3.connect('NN_features.db')
IMAGE_PATH="./../../../public/images"
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/16")
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS clip(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
clip_features BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM clip WHERE id=(?))'''
cursor.execute(query,(id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM clip WHERE id=(?)'''
cursor.execute(query,(id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM clip'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:el[0],all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id,clip_features):
cursor = conn.cursor()
query = '''INSERT INTO clip(id, clip_features) VALUES (?,?)'''
cursor.execute(query,(id,clip_features))
conn.commit()
def sync_db():
file_names=listdir(IMAGE_PATH)
ids_in_db=set(get_all_ids())
for file_name in file_names:
file_id=int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) #Fix this
print(f"deleting {id}")
def get_features(image):
image = preprocess(image).unsqueeze(0).to(device)
with torch.no_grad():
image_features = model.encode_image(image)
image_features /= image_features.norm(dim=-1, keepdim=True)
return image_features.numpy()
file_names=listdir(IMAGE_PATH)
create_table()
sync_db()
new_images=[]
for file_name in file_names:
file_id=int(file_name[:file_name.index('.')])
if check_if_exists_by_id(file_id):
continue
new_images.append(file_name)
def calc_nn_features(file_name):
file_id=int(file_name[:file_name.index('.')])
img_path=IMAGE_PATH+"/"+file_name
try:
query_image=Image.open(img_path)
except:
print(f'error reading {img_path}')
return None
image_features=get_features(query_image)
image_features_bin=adapt_array(image_features)
return (file_id,image_features_bin)
new_images=[new_images[i:i + 5000] for i in range(0, len(new_images), 5000)]
for batch in new_images:
batch_features=[]
for file_name in tqdm(batch):
batch_features.append(calc_nn_features(file_name))
batch_features= [i for i in batch_features if i] #remove None's
print("pushing data to db")
conn.executemany('''INSERT INTO clip(id, clip_features) VALUES (?,?)''', batch_features)
conn.commit()

155
ambience/modules/nn/clip_tag_images.py
View File

@ -1,155 +0,0 @@
import torch
import clip
import sqlite3
import io
import numpy as np
from os import listdir
import json
conn = sqlite3.connect('NN_features.db')
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32")
IMAGE_PATH="./../../../public/images"
ID_TAGS_ARR=[]
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM clip'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:el[0],all_rows))
def get_all_data():
cursor = conn.cursor()
query = '''
SELECT id, clip_features
FROM clip
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:{"image_id":el[0],"features":convert_array(el[1])},all_rows))
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM clip WHERE id=(?)'''
cursor.execute(query,(id,))
conn.commit()
def sync_db():
ids_in_db=set(get_all_ids())
file_names=listdir(IMAGE_PATH)
for file_name in file_names:
file_id=int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) #Fix this
print(f"deleting {id}")
print("db synced")
def tag_images():
image_data=get_all_data()
features=[]
ids=[]
for image in image_data:
features.append(image['features'])
ids.append(image['image_id'])
features=np.array(features).squeeze()
tags1=["winter", "summer", "spring", "autumn"]
tags2 = ["midnight", "noon","morning","afternoon","evening","night","dawn","dusk","sunrise","sunset"]
tags3=["sea","ocean"]
for x in tags3[:]:
tags3.append(f"{x} in the background")
tags4=["tree","forest"]
for x in tags4[:]:
tags4.append(f"{x} in the background")
tags5=["mountain","mountains"]
for x in tags5[:]:
tags5.append(f"{x} in the background")
# tags6=["cloudy sky","clear sky","uniform sky"]
tags6=["cumulus",
"altocumulus",
"stratocumulus",
"cumulonimbus",
"cirrocumulus",
"stratus",
"altostratus",
"nimbostratus",
"cirrus",
"cirrostratus",
]
tags7=["outdoors","indoors"]
tags_single=["fog","desert","cloudy","grass","grass field","beach","moon in the sky","sun in the sky","sunlight","body of water","stones","pathway"]
tags1_l=len(tags1)
tags2_l=len(tags2)
tags3_l=len(tags3)
tags4_l=len(tags4)
tags5_l=len(tags5)
tags6_l=len(tags6)
tags7_l=len(tags7)
tags_single_l=len(tags_single)
tags12_l=tags1_l+tags2_l
tags123_l=tags12_l+tags3_l
tags1234_l=tags123_l+tags4_l
tags12345_l=tags1234_l+tags5_l
tags123456_l=tags12345_l+tags6_l
tags1234567_l=tags123456_l+tags7_l
tags1234567s_l=tags1234567_l+tags_single_l
all_tags=[]
for tags in [tags1,tags2,tags3,tags4,tags5,tags6,tags7,tags_single]:
for el in tags:
# all_tags.append(el)
all_tags.append(f"a photo of a {el}")
text = clip.tokenize(all_tags).to(device)
with torch.no_grad():
text_features = model.encode_text(text)
text_features /= text_features.norm(dim=-1, keepdim=True)
text_features=text_features.cpu().numpy()
all_similarity = (100* features @ text_features.T)
print(all_similarity.shape)
for idx in range(len(ids)):
similarity=all_similarity[idx]
tags=[]
tags1_index_max = max(range(0,tags1_l), key=similarity.__getitem__)
if similarity[tags1_index_max]>=23:
tags.append(tags1[tags1_index_max])
tags2_index_max = max(range(tags1_l,tags2_l), key=similarity.__getitem__)
if similarity[tags2_index_max]>=23:
tags.append(tags2[tags2_index_max-tags1_l])
tags3_index_max = max(range(tags12_l,tags123_l), key=similarity.__getitem__)
if similarity[tags3_index_max]>=23:
tags.append(tags3[tags3_index_max-tags12_l])
tags4_index_max = max(range(tags123_l,tags1234_l), key=similarity.__getitem__)
if similarity[tags4_index_max]>=23:
tags.append(tags4[tags4_index_max-tags123_l])
tags5_index_max = max(range(tags1234_l,tags12345_l), key=similarity.__getitem__)
if similarity[tags5_index_max]>=23:
tags.append(tags5[tags5_index_max-tags1234_l])
tags6_index_max = max(range(tags12345_l,tags123456_l), key=similarity.__getitem__)
if similarity[tags6_index_max]>=23:
tags.append(tags6[tags6_index_max-tags12345_l])
tags7_index_max = max(range(tags123456_l,tags1234567_l), key=similarity.__getitem__)
if similarity[tags7_index_max]>=23:
tags.append(tags7[tags7_index_max-tags123456_l])
for i in range(tags1234567_l,tags1234567s_l):
if similarity[i]>=23:
tags.append(tags_single[i-tags1234567_l])
# print(tags)
ID_TAGS_ARR.append({"id":ids[idx],"tags":tags})
with open('./../../id_tags.txt', 'w') as outfile:
json.dump(ID_TAGS_ARR, outfile)
sync_db()
tag_images()

192
ambience/modules/nn/modules/img_tag_module.py
View File

@ -1,192 +0,0 @@
# PlacesCNN to predict the scene category, attribute, and class activation map in a single pass
# by Bolei Zhou, sep 2, 2017
# updated, making it compatible to pytorch 1.x in a hacky way
# PlacesCNN for scene classification
#
# by Bolei Zhou
# last modified by Bolei Zhou, Dec.27, 2017 with latest pytorch and torchvision (upgrade your torchvision please if there is trn.Resize error)
import io
import os
import numpy as np
import wget
import torch
from torch.autograd import Variable as V
import torchvision.models as models
from torchvision import transforms as trn
from torch.nn import functional as F
from PIL import Image
import sys
sys.path.append("./")
# hacky way to deal with the Pytorch 1.0 update
def recursion_change_bn(module):
if isinstance(module, torch.nn.BatchNorm2d):
module.track_running_stats = 1
else:
for i, (name, module1) in enumerate(module._modules.items()):
module1 = recursion_change_bn(module1)
return module
def load_labels():
# prepare all the labels
# scene category relevant
file_name_category = 'categories_places365.txt'
if not os.access(file_name_category, os.W_OK):
synset_url = 'https://raw.githubusercontent.com/csailvision/places365/master/categories_places365.txt'
wget.download(synset_url)
classes = list()
with open(file_name_category) as class_file:
for line in class_file:
classes.append(line.strip().split(' ')[0][3:])
classes = tuple(classes)
# indoor and outdoor relevant
file_name_IO = 'IO_places365.txt'
if not os.access(file_name_IO, os.W_OK):
synset_url = 'https://raw.githubusercontent.com/csailvision/places365/master/IO_places365.txt'
wget.download(synset_url)
with open(file_name_IO) as f:
lines = f.readlines()
labels_IO = []
for line in lines:
items = line.rstrip().split()
labels_IO.append(int(items[-1]) -1) # 0 is indoor, 1 is outdoor
labels_IO = np.array(labels_IO)
# scene attribute relevant
file_name_attribute = 'labels_sunattribute.txt'
if not os.access(file_name_attribute, os.W_OK):
synset_url = 'https://raw.githubusercontent.com/csailvision/places365/master/labels_sunattribute.txt'
wget.download(synset_url)
with open(file_name_attribute) as f:
lines = f.readlines()
labels_attribute = [item.rstrip() for item in lines]
file_name_W = 'W_sceneattribute_wideresnet18.npy'
if not os.access(file_name_W, os.W_OK):
synset_url = 'http://places2.csail.mit.edu/models_places365/W_sceneattribute_wideresnet18.npy'
wget.download(synset_url)
W_attribute = np.load(file_name_W)
return classes, labels_IO, labels_attribute, W_attribute
features_blob=None
def hook_feature(module, input, output):
global features_blob
features_blob=np.squeeze(output.data.cpu().numpy())
def load_model_wideresnet18():
# this model has a last conv feature map as 14x14
model_file = 'wideresnet18_places365.pth.tar'
if not os.access(model_file, os.W_OK):
wget.download("http://places2.csail.mit.edu/models_places365/"+ model_file)
wget.download("https://raw.githubusercontent.com/csailvision/places365/master/wideresnet.py")
import wideresnet
model = wideresnet.resnet18(num_classes=365)
checkpoint = torch.load(model_file, map_location=lambda storage, loc: storage)
state_dict = {str.replace(k,'module.',''): v for k,v in checkpoint['state_dict'].items()}
model.load_state_dict(state_dict)
# hacky way to deal with the upgraded batchnorm2D and avgpool layers...
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
model.avgpool = torch.nn.AvgPool2d(kernel_size=14, stride=1, padding=0)
model.eval()
# hook the feature extractor
features_names = ['layer4','avgpool'] # this is the last conv layer of the resnet
for name in features_names:
model._modules.get(name).register_forward_hook(hook_feature)
return model
def returnTF_wide_rs_18():
# load the image transformer
tf = trn.Compose([
trn.Resize((224,224)),
trn.ToTensor(),
trn.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
return tf
def returnTF_rs50():
# load the image transformer
tf = trn.Compose([
trn.Resize((256,256)),
trn.CenterCrop(224),
trn.ToTensor(),
trn.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
return tf
def load_model_resnet50():
arch = 'resnet50'
# load the pre-trained weights
model_file = '%s_places365.pth.tar' % arch
if not os.access(model_file, os.W_OK):
weight_url = 'http://places2.csail.mit.edu/models_places365/' + model_file
wget.download(weight_url)
model = models.__dict__[arch](num_classes=365)
checkpoint = torch.load(model_file, map_location=lambda storage, loc: storage)
state_dict = {str.replace(k,'module.',''): v for k,v in checkpoint['state_dict'].items()}
model.load_state_dict(state_dict)
model.eval()
return model
classes, labels_IO, labels_attribute, W_attribute = load_labels()
wideresnet18=load_model_wideresnet18()
tf_wideresnet18=returnTF_wide_rs_18()
def wideresnet18_tag(img):
tags=[]
input_img_wideresnet18 = V(tf_wideresnet18(img).unsqueeze(0))
logit = wideresnet18.forward(input_img_wideresnet18)
h_x = F.softmax(logit, 1).data.squeeze()
_, idx = h_x.sort(0, True)
idx = idx.numpy()
responses_attribute = W_attribute.dot(features_blob)
idx_a = np.argwhere(responses_attribute > 0).flatten()
for x in idx_a:
tags.append(labels_attribute[x])
return tags
# print(idx_a)
# print('--SCENE ATTRIBUTES:')
# for x in idx_a:
# print(f"{labels_attribute[x]}: {responses_attribute[x]}")
# print(', '.join([labels_attribute[idx_a[i]] for i in range(len(idx_a))]))
resnet50 =load_model_resnet50()
tf_centre_crop=returnTF_rs50()
def resnet50_tag(img):
tags=[]
input_img_resnet50 = V(tf_centre_crop(img).unsqueeze(0))
logit = resnet50.forward(input_img_resnet50)
h_x = F.softmax(logit, 1).data.squeeze()
probs, idx = h_x.sort(0, True)
io_image = np.mean(labels_IO[idx[:10]]) # vote for the indoor or outdoor
if io_image < 0.5:
tags.append("indoor")
else:
tags.append("outdoor")
# output the prediction
for i in range(len(probs)):
if probs[i] > 0.2:
# print(probs[i])
# print(classes[idx[i]])
tags.append(classes[idx[i]])
return tags
def tag(img):
if isinstance(img, str):
img = Image.open(img).convert('RGB')
else:
img = Image.open(io.BytesIO(img)).convert("RGB")
tags1=wideresnet18_tag(img)
tags2=resnet50_tag(img)
img.close()
all_tags=list(set(tags1 + tags2))
return all_tags

192
ambience/modules/nn/nn_web.py
View File

@ -1,192 +0,0 @@
import uvicorn
if __name__ == '__main__':
uvicorn.run('nn_web:app', host='127.0.0.1', port=33334, log_level="info")
from os import listdir,getcwd,path,chdir
old_cwd=getcwd()
chdir(path.join(getcwd(),"modules"))
from modules.img_tag_module import tag
chdir(old_cwd)
import torch
from pydantic import BaseModel
from fastapi import FastAPI, File,Form, HTTPException, Response, status
import clip
import numpy as np
from PIL import Image
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/16")
IMAGE_PATH="./../../../public/images"
import sqlite3
import io
conn = sqlite3.connect('NN_features.db')
import hnswlib
dim=512
index = hnswlib.Index(space='l2', dim=dim)
index.init_index(max_elements=50000, ef_construction=200, M=32)
def init_index():
image_data=get_all_data()
features=[]
ids=[]
for image in image_data:
features.append(image['features'])
ids.append(image['image_id'])
ids=np.array(ids)
features=np.array(features).squeeze()
if(len(features)!=0):
index.add_items(features,ids)
print("Index is ready")
def read_img_file(image_data):
img = Image.open(io.BytesIO(image_data)).convert("RGB")
return img
def get_features(image_buffer):
image = preprocess(read_img_file(image_buffer)).unsqueeze(0).to(device)
with torch.no_grad():
image_features = model.encode_image(image)
image_features /= image_features.norm(dim=-1, keepdim=True)
return image_features.numpy()
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS clip(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
clip_features BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM clip WHERE id=(?))'''
cursor.execute(query,(id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM clip WHERE id=(?)'''
cursor.execute(query,(id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM clip'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:el[0],all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def get_all_data():
cursor = conn.cursor()
query = '''
SELECT id, clip_features
FROM clip
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el:{"image_id":el[0],"features":convert_array(el[1])},all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id,clip_features):
cursor = conn.cursor()
query = '''INSERT INTO clip(id, clip_features) VALUES (?,?)'''
cursor.execute(query,(id,clip_features))
conn.commit()
def sync_db():
ids_in_db=set(get_all_ids())
file_names=listdir(IMAGE_PATH)
for file_name in file_names:
file_id=int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) #Fix this
print(f"deleting {id}")
print("db synced")
def get_text_features(text):
text_tokenized = clip.tokenize([text]).to(device)
with torch.no_grad():
text_features = model.encode_text(text_tokenized)
text_features /= text_features.norm(dim=-1, keepdim=True)
return text_features.numpy()
app = FastAPI()
@app.get("/")
async def read_root():
return {"Hello": "World"}
@app.post("/calculate_nn_features")
async def calculate_NN_features_handler(image: bytes = File(...),image_id: str = Form(...)):
features=get_features(image)
add_descriptor(int(image_id),adapt_array(features))
index.add_items(features,[int(image_id)])
return Response(status_code=status.HTTP_200_OK)
class Item_image_id(BaseModel):
image_id: int
@app.post("/delete_nn_features")
async def delete_nn_features_handler(item:Item_image_id):
delete_descriptor_by_id(item.image_id)
try:
index.mark_deleted(item.image_id)
except RuntimeError:
print(f"err: no image with id {item.image_id}")
return Response(status_code=status.HTTP_200_OK)
@app.post("/nn_get_similar_images_by_id")
async def get_similar_images_by_id_handler(item: Item_image_id):
try:
target_features = index.get_items([item.image_id])
labels, _ = index.knn_query(target_features, k=20)
return labels[0].tolist()
except RuntimeError:
raise HTTPException(status_code=500, detail="Image with this id is not found")
@app.post("/nn_get_similar_images_by_image_buffer")
async def nn_get_similar_images_by_image_buffer_handler(image: bytes = File(...)):
target_features=get_features(image)
labels, _ = index.knn_query(target_features, k=20)
return labels[0].tolist()
@app.post("/nn_get_image_tags_by_image_buffer")
async def nn_get_image_tags_by_image_buffer_handler(image: bytes = File(...)):
tags = tag(image)
return tags
class Item_query(BaseModel):
query: str
@app.post("/nn_get_similar_images_by_text")
async def find_similar_by_text_handler(item:Item_query):
text_features=get_text_features(item.query)
labels, _ = index.knn_query(text_features, k=20)
return labels[0].tolist()
print(__name__)
if __name__ == 'nn_web':
create_table()
sync_db()
init_index()

33
ambience/modules/nn/resnet_image_tagging.py
View File

@ -1,33 +0,0 @@
from os import path,listdir
import json
from tqdm import tqdm
from os import listdir,getcwd,path,chdir
old_cwd=getcwd()
chdir(path.join(getcwd(),"modules"))
from modules.img_tag_module import tag
chdir(old_cwd)
ID_TAGS_ARR=[]
TAG_ONLY_IMPORT=True
PUBLIC_IMAGE_PATH="./../../../public/images"
if TAG_ONLY_IMPORT and path.exists("./../../import_file_name_to_scenery_img_data.txt"):
IMAGE_PATH=PUBLIC_IMAGE_PATH
with open("./../../import_file_name_to_scenery_img_data.txt", encoding="utf-8") as jsonFile:
import_file_name_to_scenery_img_data = json.load(jsonFile)
scenery_img_data=list(import_file_name_to_scenery_img_data.values())
for img_data in tqdm(scenery_img_data):
all_tags=tag(f"{IMAGE_PATH}/{img_data['new_file_name']}")
ID_TAGS_ARR.append({"id":img_data["image_id"],"tags":all_tags})
else:
IMAGE_PATH=PUBLIC_IMAGE_PATH
file_names=listdir(IMAGE_PATH)
for file_name in tqdm(file_names):
file_id=int(file_name[:file_name.index('.')])
all_tags=tag(f"{IMAGE_PATH}/{file_name}")
ID_TAGS_ARR.append({"id":file_id,"tags":all_tags})
with open('./../../id_tags.txt', 'w') as outfile:
json.dump(ID_TAGS_ARR, outfile,ensure_ascii=False)

144
ambience/modules/phash/generate_import_phashes.py
View File

@ -1,144 +0,0 @@
from scipy.fft import dct
import cv2
import numpy as np
from numba import jit
from os import listdir, remove
from joblib import Parallel, delayed
import sqlite3
import io
conn = sqlite3.connect('import_phashes.db')
IMAGE_PATH = "./../../../import/images"
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS phashes(
id TEXT NOT NULL UNIQUE PRIMARY KEY,
phash BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM phashes WHERE id=(?))'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM phashes WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM phashes'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id, phash):
cursor = conn.cursor()
query = '''INSERT INTO phashes(id, phash) VALUES (?,?)'''
cursor.execute(query, (id, phash))
conn.commit()
@jit(cache=True, nopython=True)
def diff(dct, hash_size):
dctlowfreq = dct[:hash_size, :hash_size]
med = np.median(dctlowfreq)
diff = dctlowfreq > med
return diff.flatten()
def fast_phash(image, hash_size=16, highfreq_factor=4):
img_size = hash_size * highfreq_factor
image = cv2.resize(image, (img_size, img_size), interpolation=cv2.INTER_LINEAR) # cv2.INTER_AREA
dct_data = dct(dct(image, axis=0), axis=1)
return diff(dct_data, hash_size)
@jit(cache=True, nopython=True)
def bit_list_to_32_uint8(bit_list_256):
uint8_arr = []
for i in range(32):
bit_list = []
for j in range(8):
if(bit_list_256[i*8+j] == True):
bit_list.append(1)
else:
bit_list.append(0)
uint8_arr.append(bit_list_to_int(bit_list))
return np.array(uint8_arr, dtype=np.uint8)
@jit(cache=True, nopython=True)
def bit_list_to_int(bitlist):
out = 0
for bit in bitlist:
out = (out << 1) | bit
return out
def get_phash(query_image):
bit_list_256 = fast_phash(query_image)
phash = bit_list_to_32_uint8(bit_list_256)
return phash
def sync_db():
file_names = listdir(IMAGE_PATH)
ids_in_db = set(get_all_ids())
for file_name in file_names:
if file_name in ids_in_db:
ids_in_db.remove(file_name)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
def calc_phash(file_name):
img_path = IMAGE_PATH+"/"+file_name
query_image = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
if query_image is None:
print(f'error reading {img_path}')
remove(img_path)
return None
phash = get_phash(query_image)
phash_bin = adapt_array(phash)
return (file_name, phash_bin)
file_names = listdir(IMAGE_PATH)
create_table()
sync_db()
new_images = []
for file_name in file_names:
if check_if_exists_by_id(file_name):
continue
new_images.append(file_name)
new_images = [new_images[i:i + 5000] for i in range(0, len(new_images), 5000)]
for batch in new_images:
phashes = Parallel(n_jobs=-1, verbose=1)(delayed(calc_phash)(file_name) for file_name in batch)
phashes = [i for i in phashes if i] # remove None's
print("pushing data to db")
conn.executemany('''INSERT INTO phashes(id, phash) VALUES (?,?)''', phashes)
conn.commit()

149
ambience/modules/phash/generate_public_phashes.py
View File

@ -1,149 +0,0 @@
from scipy.fft import dct
import cv2
import numpy as np
from numba import jit
from os import listdir, remove
from joblib import Parallel, delayed
import sqlite3
import io
conn = sqlite3.connect('phashes.db')
IMAGE_PATH = "./../../../public/images"
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS phashes(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
phash BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM phashes WHERE id=(?))'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM phashes WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM phashes'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id, phash):
cursor = conn.cursor()
query = '''INSERT INTO phashes(id, phash) VALUES (?,?)'''
cursor.execute(query, (id, phash))
conn.commit()
def sync_db():
file_names = listdir(IMAGE_PATH)
ids_in_db = set(get_all_ids())
for file_name in file_names:
file_id = int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
@jit(cache=True, nopython=True)
def diff(dct, hash_size):
dctlowfreq = dct[:hash_size, :hash_size]
med = np.median(dctlowfreq)
diff = dctlowfreq > med
return diff.flatten()
def fast_phash(image, hash_size=16, highfreq_factor=4):
img_size = hash_size * highfreq_factor
image = cv2.resize(image, (img_size, img_size), interpolation=cv2.INTER_LINEAR) # cv2.INTER_AREA
dct_data = dct(dct(image, axis=0), axis=1)
return diff(dct_data, hash_size)
@jit(cache=True, nopython=True)
def bit_list_to_32_uint8(bit_list_256):
uint8_arr = []
for i in range(32):
bit_list = []
for j in range(8):
if(bit_list_256[i*8+j] == True):
bit_list.append(1)
else:
bit_list.append(0)
uint8_arr.append(bit_list_to_int(bit_list))
return np.array(uint8_arr, dtype=np.uint8)
@jit(cache=True, nopython=True)
def bit_list_to_int(bitlist):
out = 0
for bit in bitlist:
out = (out << 1) | bit
return out
def get_phash(query_image):
bit_list_256 = fast_phash(query_image)
phash = bit_list_to_32_uint8(bit_list_256)
return phash
def calc_phash(file_name):
file_id = int(file_name[:file_name.index('.')])
img_path = IMAGE_PATH+"/"+file_name
query_image = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
if query_image is None:
print(f'error reading {img_path}')
remove(img_path)
return None
phash = get_phash(query_image)
phash_bin = adapt_array(phash)
print(file_name)
return (file_id, phash_bin)
file_names = listdir(IMAGE_PATH)
create_table()
sync_db()
new_images = []
for file_name in file_names:
file_id = int(file_name[:file_name.index('.')])
if check_if_exists_by_id(file_id):
continue
new_images.append(file_name)
new_images = [new_images[i:i + 5000] for i in range(0, len(new_images), 5000)]
for batch in new_images:
phashes = Parallel(n_jobs=-1, verbose=1)(delayed(calc_phash)(file_name) for file_name in batch)
phashes = [i for i in phashes if i] # remove None's
print("pushing data to db")
conn.executemany('''INSERT INTO phashes(id, phash) VALUES (?,?)''', phashes)
conn.commit()

95
ambience/modules/phash/phash_dedup.py
View File

@ -1,95 +0,0 @@
import faiss
from os import remove
import numpy as np
import sqlite3
import io
from tqdm import tqdm
conn_import = sqlite3.connect('import_phashes.db')
conn = sqlite3.connect('phashes.db')
index = None
file_id_to_file_name_map = {}
IMAGE_PATH = "./../../../import/images"
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS phashes(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
phash BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def get_all_data():
cursor = conn.cursor()
query = '''
SELECT id, phash
FROM phashes
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: (el[0], convert_array(el[1])), all_rows))
def import_get_all_data():
cursor = conn_import.cursor()
query = '''
SELECT id, phash
FROM phashes
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: (el[0], convert_array(el[1])), all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def phash_reverse_search(target_features):
D, I = index.search(np.array([target_features]), 1)
similar = []
if D[0][0] <= 32:
similar.append(int(I[0][0]))
return similar
def dedup():
global index
all_data = get_all_data()
if len(all_data) == 0:
print("all_data no images. exit()")
exit()
try:
index = faiss.read_index_binary("trained.index")
except:
index = faiss.read_index_binary("trained_import.index")
image_ids = np.array([np.int64(x[0]) for x in all_data])
phashes = np.array([x[1] for x in all_data])
index.add_with_ids(phashes, image_ids)
print("Index is ready")
import_all_data = import_get_all_data()
if len(import_all_data) == 0:
print("import_all_data no images. exit()")
exit()
for x in tqdm(import_all_data):
filename = x[0]
features = x[1]
res = phash_reverse_search(features)
if len(res) != 0:
print(f'duplicate {filename} - {res}')
print(f'deleting {filename}')
remove(f'{IMAGE_PATH}/{x[0]}')
create_table()
dedup()

126
ambience/modules/phash/phash_dedup_internal.py
View File

@ -1,126 +0,0 @@
import faiss
from os import listdir, remove
import numpy as np
import imagesize
import sqlite3
import io
from tqdm import tqdm
conn = sqlite3.connect('import_phashes.db')
index = None
file_id_to_file_name_map = {}
IMAGE_PATH = "./../../../import/images"
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS phashes(
id TEXT NOT NULL UNIQUE PRIMARY KEY,
phash BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM phashes WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM phashes'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def get_all_data():
cursor = conn.cursor()
query = '''
SELECT id, phash
FROM phashes
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: (el[0], convert_array(el[1])), all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def sync_db():
ids_in_db = set(get_all_ids())
file_names = listdir(IMAGE_PATH)
for file_name in file_names:
if file_name in ids_in_db:
ids_in_db.remove(file_name)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
print("db synced")
def phash_reverse_search(target_features):
D, I = index.search(np.array([target_features]), 5)
similar = []
for i in range(5):
if D[0][i] <= 32:
similar.append(int(I[0][i]))
return similar
def dedup():
global index
all_data = get_all_data()
if len(all_data) == 0:
print("all_data no images. exit()")
exit()
index = faiss.read_index_binary("trained_import.index")
image_ids = np.arange(len(all_data), dtype=np.int64)
for i in range(len(all_data)):
file_id_to_file_name_map[i] = all_data[i][0]
phashes = np.array([x[1] for x in all_data])
index.add_with_ids(phashes, image_ids)
print("Index is ready")
deleted = []
for x in tqdm(all_data):
if x[0] in deleted:
continue
res = phash_reverse_search(x[1])
if len(res) != 0 and (not (len(res) == 1 and file_id_to_file_name_map[res[0]] == x[0])):
images_id_res = []
for img_id in res:
width, height = imagesize.get(
f'{IMAGE_PATH}/{file_id_to_file_name_map[img_id]}')
images_id_res.append((img_id, width*height))
print("===============")
print(f"duplicates of {x[0]}:")
for x in images_id_res:
print(f"{file_id_to_file_name_map[x[0]]} - {x[1]} pixels")
print("===============")
images_id_res.sort(key=lambda x: x[1], reverse=True)
# keep img with biggest resolution (skip image with most pixels)
for i in range(1, len(images_id_res)):
img_id = images_id_res[i][0]
print(f'deleting {file_id_to_file_name_map[img_id]}')
index.remove_ids(np.int64([img_id]))
deleted.append(file_id_to_file_name_map[img_id])
delete_descriptor_by_id(file_id_to_file_name_map[img_id])
remove(f'{IMAGE_PATH}/{file_id_to_file_name_map[img_id]}')
create_table()
sync_db()
dedup()

48
ambience/modules/phash/phash_import.py
View File

@ -1,48 +0,0 @@
import json
import sqlite3
conn_import = sqlite3.connect('import_phashes.db')
conn = sqlite3.connect('phashes.db')
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS phashes(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
phash BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def get_all_data():
cursor = conn_import.cursor()
query = '''
SELECT id, phash
FROM phashes
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: (el[0], el[1]), all_rows))
create_table()
try:
with open('./../../import_file_name_to_scenery_img_data.txt', encoding="utf-8") as json_file:
filename_to_img_id_map = json.load(json_file)
except:
print("import_file_name_to_scenery_img_data.txt not found")
exit()
phash_data_import = get_all_data()
img_id_phash = []
for phash_data in phash_data_import:
filename = phash_data[0]
phash = phash_data[1]
if filename in filename_to_img_id_map:
img_id = filename_to_img_id_map[filename]["image_id"]
img_id_phash.append((img_id, phash))
conn.executemany('''INSERT INTO phashes(id, phash) VALUES (?,?)''', img_id_phash)
conn.commit()

220
ambience/modules/phash/phash_web.py
View File

@ -1,220 +0,0 @@
import uvicorn
if __name__ == '__main__':
uvicorn.run('phash_web:app', host='127.0.0.1', port=33336, log_level="info")
import faiss
from pydantic import BaseModel
from fastapi import FastAPI, File, Form, Response, status
from os import listdir
import numpy as np
from scipy.fft import dct
from numba import jit
import cv2
import sqlite3
import io
conn = sqlite3.connect('phashes.db')
index = None
IMAGE_PATH = "./../../../public/images"
def init_index():
global index
try:
index = faiss.read_index_binary("trained.index")
except:
d = 32*8
quantizer = faiss.IndexBinaryFlat(d)
index = faiss.IndexBinaryIVF(quantizer, d, 1)
index.nprobe = 1
index.train(np.array([np.zeros(32)], dtype=np.uint8))
all_data = get_all_data()
image_ids = np.array([np.int64(x[0]) for x in all_data])
phashes = np.array([x[1] for x in all_data])
if len(all_data) != 0:
index.add_with_ids(phashes, image_ids)
print("Index is ready")
def read_img_file(image_data):
return np.fromstring(image_data, np.uint8)
@jit(cache=True, nopython=True)
def bit_list_to_32_uint8(bit_list_256):
uint8_arr = []
for i in range(32):
bit_list = []
for j in range(8):
if(bit_list_256[i*8+j] == True):
bit_list.append(1)
else:
bit_list.append(0)
uint8_arr.append(bit_list_to_int(bit_list))
return np.array(uint8_arr, dtype=np.uint8)
@jit(cache=True, nopython=True)
def bit_list_to_int(bitlist):
out = 0
for bit in bitlist:
out = (out << 1) | bit
return out
@jit(cache=True, nopython=True)
def diff(dct, hash_size):
dctlowfreq = dct[:hash_size, :hash_size]
med = np.median(dctlowfreq)
diff = dctlowfreq > med
return diff.flatten()
def fast_phash(resized_image, hash_size):
dct_data = dct(dct(resized_image, axis=0), axis=1)
return diff(dct_data, hash_size)
def get_phash(image_buffer, hash_size=16, highfreq_factor=4):
img_size = hash_size * highfreq_factor
query_image = cv2.imdecode(read_img_file(image_buffer), cv2.IMREAD_GRAYSCALE)
query_image = cv2.resize(query_image, (img_size, img_size), interpolation=cv2.INTER_LINEAR) # cv2.INTER_AREA
bit_list_256 = fast_phash(query_image, hash_size)
phash = bit_list_to_32_uint8(bit_list_256)
return phash
def get_phash_and_mirrored_phash(image_buffer, hash_size=16, highfreq_factor=4):
img_size = hash_size * highfreq_factor
query_image = cv2.imdecode(read_img_file(image_buffer), cv2.IMREAD_GRAYSCALE)
query_image = cv2.resize(query_image, (img_size, img_size), interpolation=cv2.INTER_LINEAR) # cv2.INTER_AREA
mirrored_query_image = cv2.flip(query_image, 1)
bit_list_256 = fast_phash(query_image, hash_size)
bit_list_256_mirrored = fast_phash(mirrored_query_image, hash_size)
phash = bit_list_to_32_uint8(bit_list_256)
mirrored_phash = bit_list_to_32_uint8(bit_list_256_mirrored)
return np.array([phash, mirrored_phash])
def create_table():
cursor = conn.cursor()
query = '''
CREATE TABLE IF NOT EXISTS phashes(
id INTEGER NOT NULL UNIQUE PRIMARY KEY,
phash BLOB NOT NULL
)
'''
cursor.execute(query)
conn.commit()
def check_if_exists_by_id(id):
cursor = conn.cursor()
query = '''SELECT EXISTS(SELECT 1 FROM phashes WHERE id=(?))'''
cursor.execute(query, (id,))
all_rows = cursor.fetchone()
return all_rows[0] == 1
def delete_descriptor_by_id(id):
cursor = conn.cursor()
query = '''DELETE FROM phashes WHERE id=(?)'''
cursor.execute(query, (id,))
conn.commit()
def get_all_ids():
cursor = conn.cursor()
query = '''SELECT id FROM phashes'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: el[0], all_rows))
def get_all_data():
cursor = conn.cursor()
query = '''
SELECT id, phash
FROM phashes
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: (el[0], convert_array(el[1])), all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def adapt_array(arr):
out = io.BytesIO()
np.save(out, arr)
out.seek(0)
return sqlite3.Binary(out.read())
def add_descriptor(id, phash):
cursor = conn.cursor()
query = '''INSERT INTO phashes(id, phash) VALUES (?,?)'''
cursor.execute(query, (id, phash))
conn.commit()
def sync_db():
ids_in_db = set(get_all_ids())
file_names = listdir(IMAGE_PATH)
for file_name in file_names:
file_id = int(file_name[:file_name.index('.')])
if file_id in ids_in_db:
ids_in_db.remove(file_id)
for id in ids_in_db:
delete_descriptor_by_id(id) # Fix this
print(f"deleting {id}")
print("db synced")
def phash_reverse_search(image_buffer):
target_features = get_phash_and_mirrored_phash(image_buffer)
D, I = index.search(target_features, 1)
print(D, I)
for i in range(2):
if D[i][0] <= 32:
return [int(I[i][0])]
return []
app = FastAPI()
@app.get("/")
async def read_root():
return {"Hello": "World"}
@app.post("/phash_reverse_search")
async def phash_reverse_search_handler(image: bytes = File(...)):
found_image = phash_reverse_search(image)
print(found_image)
return found_image
@app.post("/calculate_phash_features")
async def calculate_phash_features_handler(image: bytes = File(...), image_id: str = Form(...)):
features = get_phash(image)
add_descriptor(int(image_id), adapt_array(features))
index.add_with_ids(np.array([features]), np.int64([image_id]))
return Response(status_code=status.HTTP_200_OK)
class Item_image_id(BaseModel):
image_id: int
@app.post("/delete_phash_features")
async def delete_hist_features_handler(item: Item_image_id):
delete_descriptor_by_id(item.image_id)
index.remove_ids(np.int64([item.image_id]))
return Response(status_code=status.HTTP_200_OK)
print(__name__)
if __name__ == 'phash_web':
create_table()
sync_db()
init_index()

41
ambience/modules/phash/train_ivf.py
View File

@ -1,41 +0,0 @@
import sqlite3
import numpy as np
import io
import faiss
from math import sqrt
conn = sqlite3.connect('phashes.db')
def get_all_data():
cursor = conn.cursor()
query = '''
SELECT phash
FROM phashes
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: convert_array(el[0]), all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def train():
all_data = np.array(get_all_data())
if len(all_data) == 0:
print("No images. exit()")
exit()
d = 32*8
centroids = round(sqrt(all_data.shape[0]))
print(f'centroids: {centroids}')
quantizer = faiss.IndexBinaryFlat(d)
index = faiss.IndexBinaryIVF(quantizer, d, centroids)
index.nprobe = 8
index.train(all_data)
faiss.write_index_binary(index, "./" + "trained.index")
train()

41
ambience/modules/phash/train_ivf_import.py
View File

@ -1,41 +0,0 @@
import sqlite3
import numpy as np
import io
import faiss
from math import sqrt
conn = sqlite3.connect('import_phashes.db')
def get_all_data():
cursor = conn.cursor()
query = '''
SELECT phash
FROM phashes
'''
cursor.execute(query)
all_rows = cursor.fetchall()
return list(map(lambda el: convert_array(el[0]), all_rows))
def convert_array(text):
out = io.BytesIO(text)
out.seek(0)
return np.load(out)
def train():
all_data = np.array(get_all_data())
if len(all_data) == 0:
print("No images. exit()")
exit()
d = 32*8
centroids = round(sqrt(all_data.shape[0]))
print(f'centroids: {centroids}')
quantizer = faiss.IndexBinaryFlat(d)
index = faiss.IndexBinaryIVF(quantizer, d, centroids)
index.nprobe = 8
index.train(all_data)
faiss.write_index_binary(index, "./" + "trained_import.index")
train()

1522
ambience/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

32
ambience/package.json
View File

@ -1,32 +0,0 @@
{
"name": "ambience",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"build": "tsc --project tsconfig.server.json",
"build_server": "tsc --project tsconfig.server.json",
"start": "cross-env NODE_ENV=production node dist/server/index.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/qwertyforce/ambience.git"
},
"author": "",
"license": "ISC",
"bugs": {
"url": "https://github.com/qwertyforce/ambience/issues"
},
"homepage": "https://github.com/qwertyforce/ambience#readme",
"dependencies": {
"@types/node": "^15.3.0",
"axios": "^0.21.1",
"cross-env": "^7.0.3",
"fastify": "^3.20.1",
"fastify-formbody": "^5.1.0",
"fastify-multipart": "^4.0.7",
"fastify-reply-from": "^6.0.1",
"form-data": "^4.0.0",
"json-schema-to-ts": "^1.6.4"
}
}

142
ambience/server/helpers/image_ops.ts
View File

@ -1,142 +0,0 @@
import config from "../../config/config"
import FormData from 'form-data'
import axios from 'axios'
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
async function calculate_phash_features(image_id: number, image: Buffer) {
const form = new FormData();
form.append('image', image, { filename: 'document' }) //hack to make nodejs buffer work with form-data
form.append('image_id', image_id.toString())
const status = await axios.post(`${config.phash_microservice_url}/calculate_phash_features`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...form.getHeaders()
}
})
return status.data
}
async function calculate_akaze_features(image_id: number, image: Buffer) {
const form = new FormData();
form.append('image', image, { filename: 'document' }) //hack to make nodejs buffer work with form-data
form.append('image_id', image_id.toString())
const status = await axios.post(`${config.akaze_microservice_url}/calculate_akaze_features`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...form.getHeaders()
}
})
return status.data
}
async function calculate_nn_features(image_id: number, image: Buffer) {
const form = new FormData();
form.append('image', image, { filename: 'document' }) //hack to make nodejs buffer work with form-data
form.append('image_id', image_id.toString())
const status = await axios.post(`${config.nn_microservice_url}/calculate_nn_features`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...form.getHeaders()
}
})
return status.data
}
async function calculate_hist_features(image_id: number, image: Buffer) {
const form = new FormData();
form.append('image', image, { filename: 'document' }) //hack to make nodejs buffer work with form-data
form.append('image_id', image_id.toString())
const status = await axios.post(`${config.hist_microservice_url}/calculate_hist_features`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...form.getHeaders()
}
})
return status.data
}
async function phash_reverse_search(image: Buffer) {
try {
const form = new FormData();
form.append('image', image, { filename: 'document' }) //hack to make nodejs buffer work with form-data
const status = await axios.post(`${config.phash_microservice_url}/phash_reverse_search`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...form.getHeaders()
}
})
return status.data
} catch (err) {
console.log(err)
return []
}
}
async function akaze_reverse_search(image: Buffer) {
try {
const form = new FormData();
form.append('image', image, { filename: 'document' }) //hack to make nodejs buffer work with form-data
const status = await axios.post(`${config.akaze_microservice_url}/akaze_reverse_search`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...form.getHeaders()
}
})
return status.data
} catch (err) {
console.log(err)
return []
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
async function delete_akaze_features_by_id(image_id: number) {
const status = await axios.post(`${config.akaze_microservice_url}/delete_akaze_features`, { image_id: image_id })
return status.data
}
async function delete_nn_features_by_id(image_id: number) {
const status = await axios.post(`${config.nn_microservice_url}/delete_nn_features`, { image_id: image_id })
return status.data
}
async function delete_hist_features_by_id(image_id: number) {
const status = await axios.post(`${config.hist_microservice_url}/delete_hist_features`, { image_id: image_id })
return status.data
}
async function delete_phash_features_by_id(image_id: number) {
const status = await axios.post(`${config.phash_microservice_url}/delete_phash_features`, { image_id: image_id })
return status.data
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
async function calculate_all_image_features(image_id: number, image_buffer: Buffer) {
return Promise.allSettled([
calculate_akaze_features(image_id, image_buffer),
calculate_nn_features(image_id, image_buffer),
calculate_hist_features(image_id, image_buffer),
calculate_phash_features(image_id, image_buffer),
])
}
async function delete_all_image_features(image_id: number) {
return Promise.allSettled([
delete_akaze_features_by_id(image_id),
delete_nn_features_by_id(image_id),
delete_hist_features_by_id(image_id),
delete_phash_features_by_id(image_id)
])
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
export default { calculate_all_image_features, delete_all_image_features, phash_reverse_search, akaze_reverse_search }

91
ambience/server/index.ts
View File

@ -1,91 +0,0 @@
import config from './../config/config'
import fastifyMultipart from 'fastify-multipart'
import fastify from 'fastify'
import formBodyPlugin from 'fastify-formbody'
import fastifyReplyFrom from 'fastify-reply-from'
const server = fastify()
const port = config.server_port
function combineURLs(baseURL: string, relativeURL: string) { //https://stackoverflow.com/a/49966753
return relativeURL
? baseURL.replace(/\/+$/, '') + '/' + relativeURL.replace(/^\/+/, '')
: baseURL;
}
import calculate_all_image_features from "./routes/calculate_all_image_features"
import delete_all_image_features from "./routes/delete_all_image_features"
import reverse_search from "./routes/reverse_search"
server.register(async function (app) {
app.register(formBodyPlugin)
app.register(fastifyMultipart, {
attachFieldsToBody: true,
sharedSchemaId: '#mySharedSchema',
limits: {
fieldNameSize: 100, // Max field name size in bytes
fieldSize: 1000, // Max field value size in bytes
fields: 10, // Max number of non-file fields
fileSize: 50000000, // For multipart forms, the max file size in bytes //50MB
files: 1, // Max number of file fields
headerPairs: 2000 // Max number of header key=>value pairs
}
})
app.post("/calculate_all_image_features", calculate_all_image_features)
app.post("/delete_all_image_features", delete_all_image_features)
app.post("/reverse_search", reverse_search)
})
///////////////////////////////////////////////////////////////////////////////////////////////PROXY
server.register(fastifyReplyFrom)
server.addContentTypeParser('multipart/form-data', function (_request, payload, done) {
done(null, payload) //https://github.com/fastify/help/issues/67
})
const akaze_routes = ['/akaze_reverse_search', '/calculate_akaze_features', '/delete_akaze_features']
akaze_routes.forEach((r) => server.post(r, async (_req, res) => {
try {
res.from(combineURLs(config.akaze_microservice_url, r))
} catch (err) {
console.log(err)
res.status(500).send('Akaze microservice is down')
}
}))
const nn_routes = ['/nn_get_similar_images_by_image_buffer', '/nn_get_similar_images_by_text',
'/nn_get_similar_images_by_id', '/calculate_nn_features', '/delete_nn_features', '/nn_get_image_tags_by_image_buffer']
nn_routes.forEach((r) => server.post(r, async (_req, res) => {
try {
res.from(combineURLs(config.nn_microservice_url, r))
} catch (err) {
console.log(err)
res.status(500).send('NN microservice is down')
}
}))
const hist_routes = ['/hist_get_similar_images_by_image_buffer', '/hist_get_similar_images_by_id', '/calculate_hist_features', '/delete_hist_features']
hist_routes.forEach((r) => server.post(r, async (_req, res) => {
try {
res.from(combineURLs(config.hist_microservice_url, r))
} catch (err) {
console.log(err)
res.status(500).send('HIST microservice is down')
}
}))
const phash_routes = ['/phash_reverse_search', '/calculate_phash_features', '/delete_phash_features']
phash_routes.forEach((r) => server.post(r, async (_req, res) => {
try {
res.from(combineURLs(config.phash_microservice_url, r))
} catch (err) {
console.log(err)
res.status(500).send('Phash microservice is down')
}
}))
////////////////////////////////////////////////////////////////////////////////////////////////////////////
server.listen(port, "127.0.0.1", function (err, address) {
if (err) {
console.error(err)
process.exit(1)
}
console.log(`server listening on ${address}`)
})

38
ambience/server/routes/calculate_all_image_features.ts
View File

@ -1,38 +0,0 @@
import { FastifyRequest, FastifyReply } from "fastify"
import { FromSchema } from "json-schema-to-ts";
import image_ops from "./../helpers/image_ops"
const body_schema_calculate_all_image_features = {
type: 'object',
properties: {
image: { $ref: '#mySharedSchema' },
image_id: {
type: "object",
properties: {
value: { type: 'string' }
}
},
},
required: ['image', 'image_id'],
} as const;
async function calculate_all_image_features(req: FastifyRequest<{ Body: FromSchema<typeof body_schema_calculate_all_image_features> }>, res: FastifyReply) {
let image_buffer: Buffer;
try {
image_buffer = await (req as any).body.image.toBuffer()
} catch (err) {
return res.status(500).send()
}
if (req.body.image_id.value) {
const image_id = parseInt(req.body.image_id.value)
const results = await image_ops.calculate_all_image_features(image_id, image_buffer)
console.log(results)
res.send(results)
}
}
export default {
schema: {
body: body_schema_calculate_all_image_features
},
handler: calculate_all_image_features
}

23
ambience/server/routes/delete_all_image_features.ts
View File

@ -1,23 +0,0 @@
import { FastifyRequest, FastifyReply } from "fastify"
import { FromSchema } from "json-schema-to-ts";
import image_ops from "./../helpers/image_ops"
const body_schema_delete_all_image_features = {
type: 'object',
properties: {
image_id: { type: "number" }
},
required: ['image_id'],
} as const;
async function delete_all_image_features(req: FastifyRequest<{ Body: FromSchema<typeof body_schema_delete_all_image_features> }>, res: FastifyReply) {
const results = await image_ops.delete_all_image_features(req.body.image_id)
console.log(results)
res.send(results)
}
export default {
schema: {
body: body_schema_delete_all_image_features
},
handler: delete_all_image_features
}

33
ambience/server/routes/reverse_search.ts
View File

@ -1,33 +0,0 @@
import { FastifyRequest, FastifyReply } from "fastify"
import { FromSchema } from "json-schema-to-ts";
import image_ops from "./../helpers/image_ops"
const body_schema_reverse_search = {
type: 'object',
properties: {
image: { $ref: '#mySharedSchema' },
},
required: ['image'],
} as const;
async function reverse_search(req: FastifyRequest<{ Body: FromSchema<typeof body_schema_reverse_search> }>, res: FastifyReply) {
let image_buffer: Buffer;
try {
image_buffer = await (req as any).body.image.toBuffer()
} catch (err) {
return res.status(500).send()
}
const phash_found = await image_ops.phash_reverse_search(image_buffer)
if (phash_found.length !== 0) {
res.send(phash_found)
} else {
const akaze_found = await image_ops.akaze_reverse_search(image_buffer)
res.send(akaze_found)
}
}
export default {
schema: {
body: body_schema_reverse_search
},
handler: reverse_search
}

26
ambience/tsconfig.json
View File

@ -1,26 +0,0 @@
{
"compilerOptions": {
"target": "esnext",
"module": "esnext",
"jsx": "preserve",
"lib": ["dom", "es2020","ESNext.String"],
"baseUrl": ".",
"moduleResolution": "node",
"strict": true,
"allowJs": true,
"noEmit": true,
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"skipLibCheck": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"isolatedModules": true,
"removeComments": false,
"preserveConstEnums": true,
"sourceMap": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true
},
"exclude": ["dist", ".next", "out", "next.config.js"],
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"]
}

11
ambience/tsconfig.server.json
View File

@ -1,11 +0,0 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"module": "commonjs",
"outDir": "dist",
"target": "esnext",
"isolatedModules": false,
"noEmit": false
},
"include": ["server/**/*.ts"]
}

45
next.config.js
View File

@ -1,8 +1,8 @@
/* eslint-disable @typescript-eslint/no-var-requires */
require('sharp') //https://github.com/lovell/sharp/issues/2655#issuecomment-815684743
const zlib = require("zlib")
const CompressionPlugin = require("compression-webpack-plugin")
const use_brotli = false
// const zlib = require("zlib")
// const CompressionPlugin = require("compression-webpack-plugin")
// const use_brotli = false
module.exports = {
poweredByHeader: false,
compress: false,
@ -16,29 +16,30 @@ module.exports = {
}
]
},
webpack: (config, { isServer }) => {
if (use_brotli && !isServer) {
config.plugins.push(
new CompressionPlugin({
filename: "[path][base].br",
algorithm: "brotliCompress",
test: /\.(js|css|html)$/,
compressionOptions: {
params: {
[zlib.constants.BROTLI_PARAM_QUALITY]: 1,
},
}
})
)
}
return config
},
env: { //https://nextjs.org/docs/api-reference/next.config.js/environment-variables
recaptcha_site_key: "6LcqV9QUAAAAAEybBVr0FWnUnFQmOVxGoQ_Muhtb",
api_domain: "http://localhost/public_api",
reverse_search_url: "http://localhost",
domain: "http://localhost",
ipns: "ipns.scenery.cx"
},
distDir: '_next'
}
}
// , Doesn't work for some reason
// webpack: (config, { isServer }) => {
// if (use_brotli && !isServer) {
// config.plugins.push(
// new CompressionPlugin({
// filename: "[path][base].br",
// algorithm: "brotliCompress",
// test: /\.(js|css|html)$/,
// compressionOptions: {
// params: {
// [zlib.constants.BROTLI_PARAM_QUALITY]: 1,
// },
// }
// })
// )
// }
// return config
// },

2
nodemon.json
View File

@ -1,5 +1,5 @@
{
"watch": ["server"],
"watch": ["src/server"],
"exec": "npx ts-node --project tsconfig.server.json src/server/index.ts",
"ext": "js ts"
}

37
package-lock.json generated
View File

@ -23,6 +23,7 @@
"@mui/icons-material": "^5.8.4",
"@mui/material": "^5.10.0",
"@types/bcrypt": "^5.0.0",
"@types/cli-progress": "^3.11.0",
"@types/grecaptcha": "^3.0.4",
"@types/node": "^18.7.3",
"@types/nodemailer": "^6.4.5",
@ -32,6 +33,7 @@
"@typescript-eslint/parser": "^5.33.0",
"axios": "^0.27.2",
"bcrypt": "^5.0.1",
"cli-progress": "^3.11.2",
"clsx": "^1.2.1",
"compression-webpack-plugin": "^10.0.0",
"connect-mongo": "^4.6.0",
@ -1382,6 +1384,14 @@
"@types/node": "*"
}
},
"node_modules/@types/cli-progress": {
"version": "3.11.0",
"resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.0.tgz",
"integrity": "sha512-XhXhBv1R/q2ahF3BM7qT5HLzJNlIL0wbcGyZVjqOTqAybAnsLisd7gy1UCyIqpL+5Iv6XhlSyzjLCnI2sIdbCg==",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/eslint": {
"version": "8.4.5",
"resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.5.tgz",
@ -2479,6 +2489,17 @@
"node": ">=6.0"
}
},
"node_modules/cli-progress": {
"version": "3.11.2",
"resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
"integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
"dependencies": {
"string-width": "^4.2.3"
},
"engines": {
"node": ">=4"
}
},
"node_modules/clsx": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz",
@ -7798,6 +7819,14 @@
"@types/node": "*"
}
},
"@types/cli-progress": {
"version": "3.11.0",
"resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.0.tgz",
"integrity": "sha512-XhXhBv1R/q2ahF3BM7qT5HLzJNlIL0wbcGyZVjqOTqAybAnsLisd7gy1UCyIqpL+5Iv6XhlSyzjLCnI2sIdbCg==",
"requires": {
"@types/node": "*"
}
},
"@types/eslint": {
"version": "8.4.5",
"resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.5.tgz",
@ -8604,6 +8633,14 @@
"integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==",
"peer": true
},
"cli-progress": {
"version": "3.11.2",
"resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
"integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
"requires": {
"string-width": "^4.2.3"
}
},
"clsx": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz",

5
package.json
View File

@ -8,6 +8,9 @@
"build_server": "tsc --project tsconfig.server.json",
"start": "cross-env NODE_OPTIONS=--openssl-legacy-provider NODE_ENV=production node dist/server/index.js",
"start_dev": "cross-env NODE_ENV=development node dist/server/index.js",
"import_without_check": "cross-env NODE_ENV=development node dist/server/bulk_import_images/bulk_import_images_without_check.js",
"import_tags": "cross-env NODE_ENV=development node dist/server/bulk_import_images/bulk_import_tags.js",
"import_captions": "cross-env NODE_ENV=development node dist/server/bulk_import_images/bulk_import_captions.js",
"pm2_start_ms": "pm2 start ./ambience/microservices.config.js",
"pm2_start": "pm2 start ./ambience/microservices.config.js && pm2 start"
},
@ -27,6 +30,7 @@
"@mui/icons-material": "^5.8.4",
"@mui/material": "^5.10.0",
"@types/bcrypt": "^5.0.0",
"@types/cli-progress": "^3.11.0",
"@types/grecaptcha": "^3.0.4",
"@types/node": "^18.7.3",
"@types/nodemailer": "^6.4.5",
@ -36,6 +40,7 @@
"@typescript-eslint/parser": "^5.33.0",
"axios": "^0.27.2",
"bcrypt": "^5.0.1",
"cli-progress": "^3.11.2",
"clsx": "^1.2.1",
"compression-webpack-plugin": "^10.0.0",
"connect-mongo": "^4.6.0",

4
src/components/AppBar.tsx
View File

@ -147,7 +147,7 @@ function MobileMenu(props: any) {
<div className={classes.switch}>
<span>tags</span>
<Switch color="secondary" checked={props.semanticModeChecked} onChange={props.toggleSemanticModeChecked} />
<span>semantic<sub className={classes.sub}>beta</sub></span>
<span>semantic</span>
</div>
</MenuItem>
<MenuItem className={classes.sectionMobile} component={Link} color="inherit" aria-label="reverse_search" href={`${process.env.domain}/reverse_search`}>
@ -199,7 +199,7 @@ export default function DenseAppBar() {
<div className={classes.switch}>
<span>tags</span>
<Switch color="secondary" checked={semanticModeChecked} onChange={toggleSemanticModeChecked} />
<span>semantic<sub className={classes.sub}>beta</sub></span>
<span>semantic</span>
</div>
<IconButton
component={Link}

2
src/components/GalleryWrapper.tsx
View File

@ -16,7 +16,7 @@ export default function GalleryWrapper(props: { photos: PhotoInterface[] }) {
console.log(`typeof window !== "undefined" ${typeof window !== "undefined"}`)
const _photos = []
for (const photo of props.photos) {//"https://ipfs.io/ipns/ipns.scenery.cx"
const new_photo = { src: `http://127.0.0.1:8080/ipns/${process.env.ipns}${photo.src}`, key: photo.key, width: photo.width, height: photo.height }
const new_photo = { src: `http://127.0.0.1:8080/ipns/${process.env.ipns}${photo.src}`, key: photo.key, width: photo.width, height: photo.height,title:photo.title }
_photos.push(new_photo)
}
setPhotos(_photos)

12
src/config/config.ts
View File

@ -1,24 +1,22 @@
import path from "path"
const server_config = {
domain: "http://localhost",
domain: "http://127.0.0.1",
recaptcha_secret_key: "6LcqV9QUAAAAAOA18kbCEWRBhF4g4LjSTaFRVe9P",
GOOGLE_CLIENT_ID: "1006819405532-0tm9sghd6nvnpc3djf9pbrdppminbdjf.apps.googleusercontent.com",
GOOGLE_CLIENT_SECRET: "7S3KdJSNRYwkfe47dHtrJO0M",
GOOGLE_REDIRECT_URI: "http://localhost/auth/google/callback",
GOOGLE_REDIRECT_URI: "http://127.0.0.1/auth/google/callback",
GITHUB_CLIENT_ID: "d4f2879aafb5bfac8dec",
GITHUB_CLIENT_SECRET: "a2b8462d6cefb17339f4b730578db280b65e84ad",
GITHUB_REDIRECT_URI: "http://localhost/auth/github/callback",
GITHUB_REDIRECT_URI: "http://127.0.0.1/auth/github/callback",
gmail_user: "auth.test.reg.email@gmail.com",
gmail_password: "sbuLBh9rAV8XD2",
mongodb_url: "mongodb://127.0.0.1/",
server_port: "80",
session_secret: "ghuieorifigyfuu9u3i45jtr73490548t7ht",
root_path: path.join(__dirname, "..", ".."),
deviant_art_client_id: "client_id",
deviant_art_client_secret: "client_secret",
ambience_microservice_url: "http://localhost:44444",
ambience_microservice_url: "http://127.0.0.1:44444",
use_backup_file_server: false,
backup_file_server_url: "http://localhost:8787",
backup_file_server_url: "http://127.0.0.1:8787",
optimize_images: true,
}
export default server_config

11
src/pages/image/[id].tsx
View File

@ -6,6 +6,7 @@ import Grid from '@mui/material/Grid'
import LabelIcon from '@mui/icons-material/Label'
import CalendarTodayIcon from '@mui/icons-material/CalendarToday'
import AspectRatioIcon from '@mui/icons-material/AspectRatio'
import DescriptionIcon from '@mui/icons-material/Description'
import LinkIcon from '@mui/icons-material/Link'
import { GetServerSideProps } from 'next'
import db_ops from '../../server/helpers/db_ops'
@ -51,6 +52,7 @@ interface ImageProps {
similar_by_tags_link: string,
similar_by_color_link: string,
visually_similar_link: string,
caption: string
// upscaled: string,
}
export default function Image(props: ImageProps) {
@ -75,7 +77,7 @@ export default function Image(props: ImageProps) {
<Grid item xs={12} md={8}>
<Paper className={classes.paper}>
<a href={photoSrc} target="_blank" rel="noreferrer">
<img className={classes.responsive} src={photoSrc} />
<img className={classes.responsive} src={photoSrc} alt={props.caption} title={props.caption}/>
</a>
</Paper>
</Grid>
@ -107,7 +109,7 @@ export default function Image(props: ImageProps) {
</div>
<div className={classes.icon_container}>
<LinkIcon />
&nbsp;<a href={props.visually_similar_link} target="_blank" rel="noreferrer">Visually similar (Beta)</a>
&nbsp;<a href={props.visually_similar_link} target="_blank" rel="noreferrer">Visually similar</a>
</div>
{/* {((props.upscaled) ? (
<div className={classes.icon_container}>
@ -120,6 +122,10 @@ export default function Image(props: ImageProps) {
<p>&nbsp;Tags:</p>
{Tags}
</div>
<div className={classes.icon_container}>
<DescriptionIcon />
<p>&nbsp;Description: {props.caption}</p>
</div>
</Paper>
</Grid>
</Grid>
@ -149,6 +155,7 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
similar_by_tags_link: `/similar_by_tags/${img.id}`,
similar_by_color_link: `/similar_by_color/${img.id}`,
visually_similar_link: `/visually_similar/${img.id}`,
caption: img.caption
// upscaled: upscaled
}
}

3
src/pages/last_added/[page].tsx
View File

@ -77,7 +77,8 @@ export const getStaticProps: GetStaticProps = async (context) => {
src: `/thumbnails/${image.id}.jpg`,
key: `/image/${image.id}`,
width: image.width,
height: image.height
height: image.height,
title:image.caption
})
}
}

2
src/pages/public_api/index.tsx
View File

@ -1,7 +1,6 @@
import Container from '@mui/material/Container'
import Typography from '@mui/material/Typography'
import Box from '@mui/material/Box'
import AppBar from '../../components/AppBar'
import { makeStyles } from 'tss-react/mui';
import Paper from '@mui/material/Paper'
@ -17,7 +16,6 @@ export default function Api() {
const { classes } = useStyles()
return (
<div>
<AppBar />
<Container maxWidth="md">
<Box my={4}>
<Typography variant="h4" component="h1" gutterBottom>

144
src/pages/reverse_search.tsx
View File

@ -1,14 +1,13 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { useState } from 'react'
import { Fragment, useState } from 'react'
import Box from '@mui/material/Box'
import AppBar from '../components/AppBar'
import { DropzoneAreaBase } from 'mui-file-dropzone'
import Button from '@mui/material/Button'
import axios from "axios"
import { useRouter } from 'next/router'
import Backdrop from '@mui/material/Backdrop'
import CircularProgress from '@mui/material/CircularProgress'
import { makeStyles } from 'tss-react/mui';
import { makeStyles } from 'tss-react/mui'
import TextField from '@mui/material/TextField'
const useStyles = makeStyles()(() => ({
@ -24,8 +23,47 @@ const useStyles = makeStyles()(() => ({
url_text_field: {
width: "300px",
marginRight: "10px"
},
imgg: {
objectFit: "contain",
width: "150px",
height: "150px"
},
result_wrapper: {
display: "flex",
flexDirection: "row",
flexWrap: "wrap",
alignItems: "flex-start",
justifyContent: "space-around",
borderStyle: "dotted",
marginTop: "10px",
marginBottom: "10px"
},
result_wrapper_text: {
position: "relative",
marginTop: "0.3em",
marginLeft: "0.5em",
display: "inline",
width: "100%"
},
result_element_figure: {
marginLeft: "10px",
marginRight: "10px",
width: "min-content",
display: "flex",
flexWrap: "nowrap",
flexDirection: "column",
alignContent: "center",
justifyContent: "center",
alignItems: "center"
},
result_element_caption: {
fontFamily: "monospace",
fontSize: "medium",
textAlign: "center",
overflowWrap: "anywhere"
}
}));
}))
function isValidURL(url: string) {
const RegExp = /^(?:(?:(?:https?):)?\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z0-9\u00a1-\uffff][a-z0-9\u00a1-\uffff_-]{0,62})?[a-z0-9\u00a1-\uffff]\.)+(?:[a-z\u00a1-\uffff]{2,}\.?))(?::\d{2,5})?(?:[/?#]\S*)?$/i
if (RegExp.test(url)) {
@ -35,18 +73,73 @@ function isValidURL(url: string) {
}
}
function ResultElement({ result }: any) {
const { classes } = useStyles()
return (
<figure className={classes.result_element_figure}>
<a target="_blank" rel="noreferrer" href={`/image/${result["image_id"]}`}>
<img src={`/images/${result["image_id"]}.${result["ext"]}`} className={classes.imgg} />
</a>
<figcaption className={classes.result_element_caption}>
<div style={{display:"flex",flexDirection:"row",flexWrap:"wrap",justifyContent:"space-evenly"}}>
{JSON.stringify(result)}
</div>
</figcaption>
</figure>
)
}
function DisplayResults({searchResults}: any) {
const { classes } = useStyles()
let _key = 0
console.log(searchResults)
const elements = []
// searchResults = {"test":searchResults}
const keys_of_searchResults = []
if(Object.keys(searchResults).includes("unified_res")){
keys_of_searchResults.push("unified_res")
}
for(const key of Object.keys(searchResults)){
if(key!=="unified_res"){
keys_of_searchResults.push(key)
}
}
for (const key of keys_of_searchResults) {
const children = []
for (const child of searchResults[key]) {
children.push(<ResultElement key={_key++} result={child} />)
}
console.log(children)
elements.push(
<div key={_key++} className={classes.result_wrapper}>
<div className={classes.result_wrapper_text}>{key}</div>
{children}
</div>)
}
console.log(elements)
console.log(elements.length)
return (<Fragment>
{elements}
</Fragment>)
}
export default function ReverseSearch() {
const { classes } = useStyles()
const router = useRouter()
const [URL, setUrl] = useState("")
const [fileObjects, setFileObjects] = useState([])
const [open, setOpen] = useState(false)
const send_image = (token: string) => {
setOpen(true)
const [searchResults, setSearchResults] = useState<any>({})
const send_image = (token:string) => {
const formData = new FormData()
formData.append("image", (fileObjects[0] as any).file)
formData.append("g-recaptcha-response", token)
axios(`${process.env.reverse_search_url}/reverse_search`, {
const search_url = "/reverse_search"
setOpen(true)
axios(search_url, {
method: "post",
data: formData,
headers: {
@ -55,22 +148,18 @@ export default function ReverseSearch() {
timeout: 5 * 60000 //5min
}).then((resp) => {
setOpen(false)
console.log(resp.data.ids)
if(resp.data.ids===""){
alert("No images found")
}else{
router.push("/show?ids=" + resp.data.ids)
}
setSearchResults(resp.data)
console.log(resp.data)
}).catch((err) => {
setOpen(false)
console.log(err)
})
}
const proxy_get_image = (token: string, url: string) => {
const login_data = { image_url: url, 'g-recaptcha-response': token }
const get_image_data = { image_url: url,'g-recaptcha-response': token }
axios(`/proxy_get_image`, {
method: "post",
data: login_data,
data: get_image_data,
responseType: "blob"
}).then((resp) => {
const file = new File([resp.data], "image.png", { type: "image/png" })
@ -96,11 +185,12 @@ export default function ReverseSearch() {
}
const _send_image = () => {
/*global grecaptcha*/ // defined in pages/_document.tsx
grecaptcha.ready(function () {
grecaptcha.execute(process.env.recaptcha_site_key, { action: 'reverse_search' }).then(function (token) {
send_image(token)
})
})
grecaptcha.ready(function () {
grecaptcha.execute(process.env.recaptcha_site_key, { action: 'reverse_search' }).then(function (token) {
send_image(token)
})
})
// send_image()
}
const get_image_by_url = async () => {
if (!isValidURL(URL)) {
@ -121,7 +211,7 @@ export default function ReverseSearch() {
setUrl("")
return
}
} catch (err:any) {
} catch (err: any) {
console.log(err)
if (!err.response) {
grecaptcha.ready(function () {
@ -129,9 +219,10 @@ export default function ReverseSearch() {
proxy_get_image(token, URL)
})
})
// proxy_get_image(URL)
}
// alert("error")
alert("error")
}
}
@ -143,7 +234,7 @@ export default function ReverseSearch() {
</Backdrop>
<Box my={4}>
<div className={classes.url_div}>
<TextField onChange={(e:any) => setUrl(e.target.value)} value={URL}
<TextField onChange={(e) => setUrl(e.target.value)} value={URL}
className={classes.url_text_field} label="url"
placeholder="https://somesite.com/image.png" variant="outlined" size="small" />
<Button onClick={get_image_by_url} size="small" variant="outlined">Fetch</Button>
@ -167,7 +258,8 @@ export default function ReverseSearch() {
maxFileSize={49000000}
/>
</Box>
<Button onClick={() => { _send_image() }} variant="contained" color="primary">Reverse Search</Button>
<Button onClick={() => { _send_image() }} variant="contained" color="primary">Reverse Search</Button>
<DisplayResults searchResults={searchResults}/>
</div>
)
}
}

5
src/pages/search.tsx
View File

@ -77,7 +77,7 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
return ERROR
}
const found_images = []
const image_ids = await image_ops.nn_get_similar_images_by_text(context.query.q)
const image_ids = await image_ops.image_text_features_get_similar_images_by_text(context.query.q)
if (!image_ids) {
return {
props: {
@ -134,7 +134,8 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
src: `/thumbnails/${image.id}.jpg`,
key: `/image/${image.id}`,
width: image.width,
height: image.height
height: image.height,
title:image.caption
})
}
return {

5
src/pages/show.tsx
View File

@ -21,13 +21,14 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
const img_data = await db_ops.image_ops.find_image_by_id(parseInt(id))
if (img_data) { images.push(img_data) }
}
const photos: PhotoInterface[] = []
const photos = []
for (const image of images) {
photos.push({
src: `/thumbnails/${image.id}.jpg`,
key: `/image/${image.id}`,
width: image.width as number,
height: image.height as number
height: image.height as number,
title:image.caption
})
}
if (photos.length !== 0) {

7
src/pages/similar_by_color/[id].tsx
View File

@ -27,14 +27,14 @@ export default function SimilarByColor(props: SimilarByColorProps) {
export const getServerSideProps: GetServerSideProps = async (context) => {
const photos = []
if (typeof context.params?.id === "string") {
const similar_images_ids = await image_ops.hist_get_similar_images_by_id(parseInt(context.params.id))
const similar_images_ids = await image_ops.color_get_similar_images_by_id(parseInt(context.params.id))
// console.log(similar_images_ids)
if (similar_images_ids) {
const similar_images = []
for (const image_id of similar_images_ids) {
const img = await db_ops.image_ops.find_image_by_id(image_id)
if (img) {
similar_images.push({ id: img.id, width: img.width, height: img.height })
similar_images.push({ id: img.id, width: img.width, height: img.height,caption:img.caption })
}
}
for (const image of similar_images) {
@ -42,7 +42,8 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
src: `/thumbnails/${image.id}.jpg`,
key: `/image/${image.id}`,
width: image.width,
height: image.height
height: image.height,
title:image.caption
})
}
return {

6
src/pages/similar_by_tags/[id].tsx
View File

@ -25,7 +25,8 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
return {
id: el["_id"].id,
width: el["_id"].width,
height: el["_id"].height
height: el["_id"].height,
caption: el["_id"].caption,
}
})
for (const image of similar_images) {
@ -33,7 +34,8 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
src: `/thumbnails/${image.id}.jpg`,
key: `/image/${image.id}`,
width: image.width,
height: image.height
height: image.height,
title:image.caption
})
}
return {

7
src/pages/visually_similar/[id].tsx
View File

@ -19,14 +19,14 @@ export default function VisuallySimilar(props: { photos: PhotoInterface[] }) {
export const getServerSideProps: GetServerSideProps = async (context) => {
const photos = []
if (typeof context.params?.id === "string") {
const similar_images_ids = await image_ops.nn_get_similar_images_by_id(parseInt(context.params.id))
const similar_images_ids = await image_ops.image_text_features_get_similar_images_by_id(parseInt(context.params.id))
// console.log(similar_images_ids)
if (similar_images_ids) {
const similar_images = []
for (const image_id of similar_images_ids) {
const img = await db_ops.image_ops.find_image_by_id(image_id)
if (img) {
similar_images.push({ id: img.id, width: img.width, height: img.height })
similar_images.push({ id: img.id, width: img.width, height: img.height,caption:img.caption })
}
}
for (const image of similar_images) {
@ -34,7 +34,8 @@ export const getServerSideProps: GetServerSideProps = async (context) => {
src: `/thumbnails/${image.id}.jpg`,
key: `/image/${image.id}`,
width: image.width,
height: image.height
height: image.height,
title:image.caption
})
}
return {

18
src/server/bulk_import_images/bulk_import_captions.ts Normal file
View File

@ -0,0 +1,18 @@
import fs from 'fs'
import path from 'path'
import db_ops from "../helpers/db_ops"
import config from "../../config/config"
import cliProgress from "cli-progress"
const fsPromises = fs.promises
const bar1 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
async function import_tags() {
const img_captions=JSON.parse(await fsPromises.readFile(path.join(config.root_path,"import","id_caption.txt"),'utf8'))
bar1.start(img_captions.length, 0)
for (const img of img_captions) {
await db_ops.image_ops.set_caption_to_image_by_id(img.id,img.caption)
bar1.increment()
}
process.exit()
}
import_tags()

24
src/server/bulk_import_images/bulk_import_images_without_check.ts
View File

@ -2,19 +2,29 @@ import fs from 'fs'
import path from 'path'
import image_ops from "../helpers/image_ops"
import config from "../../config/config"
import db_ops from "../helpers/db_ops"
const fsPromises = fs.promises;
const PATH_TO_IMAGE_IMPORT = path.join(config.root_path, 'import', 'images')
const IMAGES = fs.readdirSync(PATH_TO_IMAGE_IMPORT)
const import_file_name_to_scenery_img_data:any={}
async function import_images() {
for (const image_file_name of IMAGES) {
const img_path=`${PATH_TO_IMAGE_IMPORT}/${image_file_name}`
const img_buffer= await fsPromises.readFile(img_path)
const img_data = await image_ops.import_image_without_check(img_buffer)
fsPromises.unlink(img_path)
import_file_name_to_scenery_img_data[image_file_name]=img_data
const img_path = `${PATH_TO_IMAGE_IMPORT}/${image_file_name}`
const img_buffer = await fsPromises.readFile(img_path)
const img_id = parseInt(path.parse(img_path).name)
if (isNaN(img_id)) {
console.log(`${path.parse(img_path).name} is not a number`)
break
}
const img_exists = await db_ops.image_ops.check_if_image_exists_by_id(img_id)
if (img_exists){
console.log(`id: ${img_id} is already in db`)
break
}
const img_data = await image_ops.import_image(img_buffer, [], "", true, img_id)
console.log(img_data)
// fsPromises.unlink(img_path)
}
await fsPromises.writeFile(path.join(config.root_path,"ambience","import_file_name_to_scenery_img_data.txt"),JSON.stringify(import_file_name_to_scenery_img_data))
process.exit()
}
import_images()

6
src/server/bulk_import_images/bulk_import_tags.ts
View File

@ -2,12 +2,16 @@ import fs from 'fs'
import path from 'path'
import db_ops from "../helpers/db_ops"
import config from "../../config/config"
import cliProgress from "cli-progress"
const fsPromises = fs.promises;
const bar1 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
async function import_tags() {
const img_tags=JSON.parse(await fsPromises.readFile(path.join(config.root_path,"ambience","id_tags.txt"),'utf8'))
const img_tags=JSON.parse(await fsPromises.readFile(path.join(config.root_path,"import","id_tags.txt"),'utf8'))
bar1.start(img_tags.length, 0)
for (const img of img_tags) {
await db_ops.image_ops.add_tags_to_image_by_id(img.id,img.tags)
bar1.increment()
}
process.exit()
}

21
src/server/bulk_import_images/dedupe_import_sha256.ts
View File

@ -1,21 +0,0 @@
import fs from 'fs'
import path from 'path'
import db_ops from "../helpers/db_ops"
import crypto_ops from "../helpers/crypto_ops"
import config from "../../config/config"
const fsPromises = fs.promises;
const PATH_TO_IMAGE_IMPORT = path.join(config.root_path, 'import', 'images')
const IMAGES = fs.readdirSync(PATH_TO_IMAGE_IMPORT)
async function import_images() {
for (const image_file_name of IMAGES) {
const img_buffer = await fsPromises.readFile(`${PATH_TO_IMAGE_IMPORT}/${image_file_name}`)
const sha256_hash = await crypto_ops.image_buffer_sha256_hash(img_buffer)
const found_img = await db_ops.image_ops.find_image_by_sha256(sha256_hash)
if (found_img) {
console.log(`${image_file_name} is a duplicate of img_id = ${found_img.id}. Deleting`)
await fsPromises.unlink(`${PATH_TO_IMAGE_IMPORT}/${image_file_name}`)
}
}
process.exit()
}
import_images()

82
src/server/deviant_art_checker/deviant_art_checker._ts
View File

@ -1,82 +0,0 @@
import config from '../../config/config';
import axios from 'axios';
import db_ops from '../helpers/db_ops';
import puppeteer from 'puppeteer';
import {promises as fs} from 'fs'
function sleep(ms:number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async function get_token(){
const params = new URLSearchParams();
params.append('grant_type', 'client_credentials');
params.append('client_id', config.deviant_art_client_id);
params.append('client_secret', config.deviant_art_client_secret);
const result = await axios.post("https://www.deviantart.com/oauth2/token", params,{withCredentials: true})
const access_token = result.data.access_token
return access_token
}
async function deviant_art_checker() {
const browser = await puppeteer.launch({ headless: true,args: ['--no-sandbox', '--disable-setuid-sandbox'] });
const images = await db_ops.image_ops.get_all_images()
let access_token = await get_token()
for (let i=0;i<images.length;i++) {
const image=images[i]
console.log(image.id)
if (image.source_url?.includes("deviantart.com")) {
await sleep(3000)
try {
const page = await browser.newPage();
const x = await page.goto(image.source_url,{waitUntil:"domcontentloaded"});
if(!x){
continue
}
const text=await x.text()
page.close()
const idx_1 = text.indexOf("DeviantArt://deviation/")
const error = text.indexOf("403 ERROR")
if(error!==-1){
console.log("403 ERROR")
await sleep(5000)
continue
}
if (idx_1 === -1) {
console.log("not_found")
continue;
}
const deviant_art_id = text.slice(idx_1 + 23, text.indexOf(`"/>`, idx_1))
const res = await axios.get(`https://www.deviantart.com/api/v1/oauth2/deviation/download/${deviant_art_id}?mature_content=true&access_token=${access_token}`,{withCredentials: true})
if(image.width*image.height<res.data.width*res.data.height){
const new_img= await axios.get(res.data.src,{responseType: 'arraybuffer'})
console.log(`${config.root_path}/public/images/${image.id}.${image.file_ext}`)
const tags=image.tags
for(let i=0;i<tags.length;i++){
if(tags[i].includes("width:") || tags[i].includes("height:")) {
tags.splice(i,1)
}
}
tags.push(`height:${res.data.height}`)
tags.push(`width:${res.data.width}`)
await fs.writeFile(`${config.root_path}/public/images/${image.id}.${image.file_ext}`,new_img.data,"binary")
await db_ops.image_ops.update_image_data_by_id(image.id,{width:res.data.width,height:res.data.height,size:res.data.filesize,tags:tags})
console.log(res.data)
}
} catch (e:any) {
console.log(`img_id ${image.id} ${e.response?.status}`)
if(e.response?.status===401){
access_token = await get_token()
}
// console.log(e.response?.data)
}
}
}
}
async function run() {
await deviant_art_checker()
process.exit()
}
run()

19
src/server/helpers/db_ops.ts
View File

@ -37,7 +37,7 @@ interface PasswordRecoveryObject {
interface Image {
id: number,
created_at: Date,
description: string
caption: string
source_url: string,
file_ext: string,
width: number,
@ -89,7 +89,7 @@ async function get_number_of_images_returned_by_search_query(query: Record<strin
}
async function batch_find_images(query: Record<string, unknown>, skip: number, limit: number) {
const data: Promise<Image[]> = IMAGES_COLLECTION.find(query).sort({ "$natural": -1 }).skip(skip).limit(limit).project<Image>({ _id: 0 }).toArray()
const data: Promise<Image[]> = IMAGES_COLLECTION.find(query).sort({ "id": -1 }).skip(skip).limit(limit).project<Image>({ _id: 0 }).toArray()
return data
}
@ -105,6 +105,10 @@ async function add_tags_to_image_by_id(id: number, tags: string[]) {
await IMAGES_COLLECTION.updateOne({ id: id }, { $addToSet: { tags: { $each: tags } } })
}
async function set_caption_to_image_by_id(id: number, caption: string) {
await IMAGES_COLLECTION.updateOne({ id: id }, { $set: { caption: caption } })
}
async function update_image_data_by_id(id: number, update: Record<string, unknown>) {
return IMAGES_COLLECTION.updateOne({ id: id }, { $set: update })
}
@ -115,8 +119,8 @@ async function get_all_images() {
}
async function get_image_file_extension_by_id(id: number) {
const img = IMAGES_COLLECTION.find({ id: id }).project({ file_ext: 1, _id: 0 }).next()
return img
const img = await IMAGES_COLLECTION.find({ id: id }).project({ file_ext: 1, _id: 0 }).next()
return img?.file_ext
}
@ -145,13 +149,13 @@ async function add_image(img: Image) {
width: img.width,
height: img.height,
author: img.author,
description: img.description,
caption: img.caption,
size: img.size,
sha256: img.sha256,
tags: img.tags,
source_url: img.source_url
}
IMAGES_COLLECTION.insertOne(image)
return IMAGES_COLLECTION.insertOne(image)
}
async function add_image_by_object(image: Record<string, unknown>) {
return IMAGES_COLLECTION.insertOne(image)
@ -195,7 +199,7 @@ async function get_images_with_similar_tags(image_id: number, limit: number) {
const x = IMAGES_COLLECTION.aggregate([
{ $unwind: "$tags" },
{ $match: { tags: { $in: target_tags } } },
{ $group: { _id: { id: "$id", height: "$height", width: "$width" }, count: { $sum: 1 } } },
{ $group: { _id: { id: "$id", height: "$height", width: "$width",caption: "$caption" }, count: { $sum: 1 } } },
{ $sort: { count: -1 } },
{ $limit: limit }
])
@ -334,6 +338,7 @@ export default {
check_if_image_exists_by_id,
update_image_data_by_id,
add_tags_to_image_by_id,
set_caption_to_image_by_id
},
password_recovery: {
update_user_password_by_id,

235
src/server/helpers/image_ops.ts
View File

@ -9,13 +9,13 @@ import { unlink as fs_unlink_callback } from 'fs'
import { promisify } from 'util'
import { exec } from 'child_process'
const exec_async = promisify(exec);
import {fromBuffer} from 'file-type'
import { fromBuffer } from 'file-type'
import path from "path"
const PATH_TO_IMAGES = path.join(config.root_path, 'public', 'images')
const PATH_TO_THUMBNAILS = path.join(config.root_path, 'public', 'thumbnails')
const PATH_TO_TEMP = path.join(config.root_path, 'temp')
const JPEGTRAN_PATH = process.platform === 'win32' ? path.join(config.root_path, "bin", "jpegtran.exe") : "jpegtran"
const OXIPNG_PATH = process.platform === 'win32' ? path.join(config.root_path, "bin", "oxipng.exe") : path.join(config.root_path, "bin", "oxipng")
const JPEGTRAN_PATH = process.platform === 'win32' ? path.join(config.root_path,"src", "bin", "jpegtran.exe") : "jpegtran"
const OXIPNG_PATH = process.platform === 'win32' ? path.join(config.root_path, "src", "bin", "oxipng.exe") : path.join(config.root_path,"src", "bin", "oxipng")
async function optimize_image(extension: string, image: Buffer) {
try {
@ -58,8 +58,11 @@ async function generate_thumbnail(image_src: Buffer | string) { //buffer or pat
}
}
async function reverse_search(image: Buffer) {
const form = new FormData();
async function reverse_search(image: Buffer,find_duplicate = false) {
const form = new FormData()
if(find_duplicate){
form.append('find_duplicate', "1")
}
form.append('image', image, { filename: 'document' }) //hack to make nodejs buffer work with form-data
try {
const res = await axios.post(`${config.ambience_microservice_url}/reverse_search`, form.getBuffer(), {
@ -72,32 +75,32 @@ async function reverse_search(image: Buffer) {
return res.data
} catch (err) {
console.log(err)
return []
return {}
}
}
async function nn_get_similar_images_by_id(image_id: number) {
async function image_text_features_get_similar_images_by_id(image_id: number) {
try {
const res = await axios.post(`${config.ambience_microservice_url}/nn_get_similar_images_by_id`, { image_id: image_id })
return res.data
const res = await axios.post(`${config.ambience_microservice_url}/image_text_features_get_similar_images_by_id`, { image_id: image_id,k:20 })
return res.data.map((el:any)=>el["image_id"])
} catch (err) {
console.log(err)
}
}
async function nn_get_similar_images_by_text(query: string) {
async function image_text_features_get_similar_images_by_text(query: string) {
try {
const res = await axios.post(`${config.ambience_microservice_url}/nn_get_similar_images_by_text`, { query: query })
return res.data
const res = await axios.post(`${config.ambience_microservice_url}/image_text_features_get_similar_images_by_text`, { text: query,k:20 })
return res.data.map((el:any)=>el["image_id"])
} catch (err) {
console.log(err)
}
}
async function hist_get_similar_images_by_id(image_id: number) {
async function color_get_similar_images_by_id(image_id: number) {
try {
const res = await axios.post(`${config.ambience_microservice_url}/hist_get_similar_images_by_id`, { image_id: image_id })
return res.data
const res = await axios.post(`${config.ambience_microservice_url}/color_get_similar_images_by_id`, { image_id: image_id,k:20 })
return res.data.map((el:any)=>el["image_id"])
} catch (err) {
console.log(err)
}
@ -122,11 +125,11 @@ async function calculate_all_image_features(image_id: number, image_buffer: Buff
}
}
async function nn_get_image_tags(image_buffer: Buffer) {
async function get_image_tags(image_buffer: Buffer): Promise<string[]> {
const form = new FormData();
form.append('image', image_buffer, { filename: 'document' }) //hack to make nodejs buffer work with form-data
try {
const similar = await axios.post(`${config.ambience_microservice_url}/nn_get_image_tags_by_image_buffer`, form.getBuffer(), {
const similar = await axios.post(`${config.ambience_microservice_url}/get_image_tags`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
@ -140,6 +143,24 @@ async function nn_get_image_tags(image_buffer: Buffer) {
}
}
async function get_image_caption(image_buffer: Buffer): Promise<string> {
const form = new FormData();
form.append('image', image_buffer, { filename: 'document' }) //hack to make nodejs buffer work with form-data
try {
const resp = await axios.post(`${config.ambience_microservice_url}/get_image_caption`, form.getBuffer(), {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...form.getHeaders()
}
})
return resp.data
} catch (err) {
console.log(err)
return ""
}
}
async function upload_data_to_backup_server(full_paths: string[], file_buffers: Buffer[]) {
const form = new FormData();
form.append('full_paths', JSON.stringify(full_paths))
@ -175,7 +196,7 @@ function get_orientation(height: number, width: number) {
}
}
async function parse_author(tags: string[]) {
function parse_author(tags: string[]) { //tags like "artist:shishkin"
for (const tag of tags) {
const idx = tag.indexOf("artist:")
if (idx === 0) { //tag starts with "artist:"
@ -185,19 +206,19 @@ async function parse_author(tags: string[]) {
return "???"
}
async function import_image(image_buffer: Buffer, tags: string[] = [], source_url = "") {
const sha256_hash = crypto_ops.image_buffer_sha256_hash(image_buffer)
const found_img = await db_ops.image_ops.find_image_by_sha256(sha256_hash)
if (found_img) {
return `Image with the same sha256 is already in the db. Image id = ${found_img.id} `
}
if (!tags.includes("bypass_dup_check")) {
const res = await reverse_search(image_buffer)
if (res.length !== 0) {
return `Image with the same phash/akaze descriptors is already in the db. Image id = ${res[0]} `
}
}
async function import_image(image_buffer: Buffer, tags: string[] = [], source_url = "", local_add = false, img_id=-1) {
try {
const sha256_hash = crypto_ops.image_buffer_sha256_hash(image_buffer)
const found_img = await db_ops.image_ops.find_image_by_sha256(sha256_hash)
if (found_img) {
return `Image with the same sha256 is already in the db. Image id = ${found_img.id} `
}
if (!local_add && !tags.includes("bypass_dup_check")) {
const res = await reverse_search(image_buffer,true)
if (res["local_features_res"] !== undefined) {
return `Similar image is already in the db. Image ids = ${JSON.stringify(res["local_features_res"])} `
}
}
const mime_type = (await fromBuffer(image_buffer))?.mime
let file_ext = ""
switch (mime_type) {
@ -218,110 +239,72 @@ async function import_image(image_buffer: Buffer, tags: string[] = [], source_ur
const size = metadata.size || 10
const height = metadata.height || 10
const width = metadata.width || 10
const orientation = get_orientation(height, width)
if (config.optimize_images) {
image_buffer = await optimize_image(file_ext, image_buffer)
}
tags.push(orientation)
const new_image_id = (await db_ops.image_ops.get_max_image_id()) + 1
const author = await parse_author(tags)
const generated_tags = await nn_get_image_tags(image_buffer)
for (const tag of generated_tags) {
tags.push(tag)
}
await db_ops.image_ops.add_image({ id: new_image_id, description: "", source_url: source_url, file_ext: file_ext, width: width, height: height, author: author, size: size, tags: [...new Set(tags)], sha256: sha256_hash, created_at: new Date() })
await fs.writeFile(`${PATH_TO_IMAGES}/${new_image_id}.${file_ext}`, image_buffer, 'binary')
const thumbnail_buffer = await generate_thumbnail(image_buffer)
if (!thumbnail_buffer) {
return "Can't generate thumbnail"
}
await fs.writeFile(`${PATH_TO_THUMBNAILS}/${new_image_id}.jpg`, thumbnail_buffer, 'binary')
const res = await calculate_all_image_features(new_image_id, image_buffer)
if (!res) {
return "Can't calculate_all_image_features"
const orientation = get_orientation(height, width)
tags.push(orientation)
if (config.optimize_images) {
image_buffer = await optimize_image(file_ext, image_buffer)
}
console.log(`Akaze calc=${res[0].status}`)
console.log(`NN calc=${res[1].status}`)
console.log(`HIST calc=${res[2].status}`)
console.log(`VP calc=${res[3].status}`)
console.log(`OK. New image_id: ${new_image_id}`)
if (config.use_backup_file_server) {
try {
await upload_data_to_backup_server([`images/${new_image_id}.${file_ext}`, `thumbnails/${new_image_id}.jpg`], [image_buffer, thumbnail_buffer])
console.log("uploaded to backup server")
} catch (err) {
console.log("backup_error")
console.log(err)
const author = parse_author(tags)
let generated_tags = []
let caption = ""
if(!local_add){
[generated_tags, caption] = (await Promise.allSettled([get_image_tags(image_buffer), get_image_caption(image_buffer)])).map((promise: any) => promise.value)
tags.push(...generated_tags)
}
const new_image_id = img_id === -1 ? (await db_ops.image_ops.get_max_image_id()) + 1 : img_id
await db_ops.image_ops.add_image({ id: new_image_id, caption, source_url, file_ext, width, height, author, size, tags: [...new Set(tags)], sha256: sha256_hash, created_at: new Date() })
await fs.writeFile(`${PATH_TO_IMAGES}/${new_image_id}.${file_ext}`, image_buffer, 'binary')
await fs.writeFile(`${PATH_TO_THUMBNAILS}/${new_image_id}.jpg`, thumbnail_buffer, 'binary')
if(!local_add){
const res = await calculate_all_image_features(new_image_id, image_buffer)
if (!res) {
return "Can't calculate_all_image_features"
}
console.log(`Akaze calc=${res[0].status}`)
console.log(`NN calc=${res[1].status}`)
console.log(`HIST calc=${res[2].status}`)
console.log(`VP calc=${res[3].status}`)
if (config.use_backup_file_server) {
try {
await upload_data_to_backup_server([`images/${new_image_id}.${file_ext}`, `thumbnails/${new_image_id}.jpg`], [image_buffer, thumbnail_buffer])
console.log("uploaded to backup server")
} catch (err) {
console.log("backup_error")
console.log(err)
}
}
}
console.log(`OK. New image_id: ${new_image_id}. local_add = ${local_add}`)
return `Success! Image id = ${new_image_id}`
} catch (error) {
console.error(error);
}
}
async function import_image_without_check(image_buffer: Buffer, tags: string[] = [], source_url = "") {
const sha256_hash = await crypto_ops.image_buffer_sha256_hash(image_buffer)
const found_img = await db_ops.image_ops.find_image_by_sha256(sha256_hash)
if (found_img) {
return `Image with the same sha256 is already in the db. Image id = ${found_img.id} `
}
try {
const mime_type = (await fromBuffer(image_buffer))?.mime
let file_ext = ""
switch (mime_type) {
case "image/png":
file_ext = "png"
break
case "image/jpeg":
file_ext = "jpg"
break
}
let metadata = await sharp(image_buffer).metadata()
if (metadata.orientation) { //rotate according to EXIF
image_buffer = await sharp(image_buffer).rotate().toBuffer()
metadata = await sharp(image_buffer).metadata()
}
const size = metadata.size || 10
const height = metadata.height || 10
const width = metadata.width || 10
const orientation = get_orientation(height, width)
if (config.optimize_images) {
image_buffer = await optimize_image(file_ext, image_buffer)
}
tags.push(orientation)
const new_image_id = (await db_ops.image_ops.get_max_image_id()) + 1
const author = await parse_author(tags)
await db_ops.image_ops.add_image({ id: new_image_id, description: "", source_url: source_url, file_ext: file_ext, width: width, height: height, author: author, size: size, tags: tags, sha256: sha256_hash, created_at: new Date() })
await fs.writeFile(`${PATH_TO_IMAGES}/${new_image_id}.${file_ext}`, image_buffer, 'binary')
const thumbnail_buffer = await generate_thumbnail(image_buffer)
if (!thumbnail_buffer) {
return "Can't generate thumbnail"
}
await fs.writeFile(`${PATH_TO_THUMBNAILS}/${new_image_id}.jpg`, thumbnail_buffer, 'binary')
// console.log(`OK. New image_id: ${new_image_id}`)
return { image_id: new_image_id, new_file_name: `${new_image_id}.${file_ext}` }
} catch (error) {
console.error(error);
}
}
async function delete_image(id: number) {
try {
const image = await db_ops.image_ops.get_image_file_extension_by_id(id)
if (!image) {
const file_ext = await db_ops.image_ops.get_image_file_extension_by_id(id) // this information is needed to delete image files
if (!file_ext) {
console.log("image_not_found")
return "not_found"
}
fs_unlink_callback(`${config.root_path}/public/images/${id}.${image.file_ext}`, function (err) {
const res = await delete_all_image_features(id)
if (!res) {
return "Can't delete all_image_features"
}
console.log(`Akaze del=${res[0].status}`)
console.log(`NN del=${res[1].status}`)
console.log(`HIST del=${res[2].status}`)
console.log(`VP del=${res[3].status}`)
console.log(`OK. Deleted image_id: ${id}`)
db_ops.image_ops.delete_image_by_id(id)
fs_unlink_callback(`${config.root_path}/public/images/${id}.${file_ext}`, function (err) {
if (err) return console.log(err);
console.log('main image deleted successfully');
});
@ -338,7 +321,7 @@ async function delete_image(id: number) {
try {
await axios.post(`${config.backup_file_server_url}/delete_files`, {
full_paths: [
`images/${id}.${image.file_ext}`, `thumbnails/${id}.jpg`]
`images/${id}.${file_ext}`, `thumbnails/${id}.jpg`]
})
console.log("deleted from backup server")
} catch (err) {
@ -347,16 +330,7 @@ async function delete_image(id: number) {
}
}
const res = await delete_all_image_features(id)
if (!res) {
return "Can't delete all_image_features"
}
console.log(`Akaze del=${res[0].status}`)
console.log(`NN del=${res[1].status}`)
console.log(`HIST del=${res[2].status}`)
console.log(`VP del=${res[3].status}`)
await db_ops.image_ops.delete_image_by_id(id)
console.log(`OK. Deleted image_id: ${id}`)
return true
} catch (error) {
console.error(error);
@ -366,11 +340,10 @@ export default {
import_image,
delete_image,
get_orientation,
nn_get_similar_images_by_id,
nn_get_similar_images_by_text,
image_text_features_get_similar_images_by_id,
image_text_features_get_similar_images_by_text,
reverse_search,
hist_get_similar_images_by_id,
color_get_similar_images_by_id,
calculate_all_image_features,
import_image_without_check,
delete_all_image_features
}

14
src/server/routes/reverse_search.ts
View File

@ -1,6 +1,7 @@
import image_ops from '../helpers/image_ops'
import sharp from 'sharp'
import { FastifyRequest, FastifyReply } from "fastify"
import db_ops from '../helpers/db_ops';
const body_schema_reverse_search = {
type: 'object',
@ -28,7 +29,7 @@ async function reverse_search(req: FastifyRequest, res: FastifyReply) {
throw "not a buffer"
}
} catch (err) {
return res.send({ ids: '' })
return res.send({})
}
const metadata = await sharp(image_buffer).metadata()
@ -36,9 +37,14 @@ async function reverse_search(req: FastifyRequest, res: FastifyReply) {
image_buffer = await sharp(image_buffer).rotate().toBuffer()
}
const ids = await image_ops.reverse_search(image_buffer)
// console.log(ids)
res.send({ ids: ids.join(',') })
const data = await image_ops.reverse_search(image_buffer)
for (const key in data) {
for (let i = 0; i < data[key].length; i++) {
const ext = await db_ops.image_ops.get_image_file_extension_by_id(data[key][i]["image_id"])
data[key][i].ext = ext
}
}
res.send(data)
}
export default {

3
src/types/photo.ts
View File

@ -2,5 +2,6 @@ export default interface Photo {
src: string,
key: string,
width: number,
height: number
height: number,
title: string
}

0
ambience/image_tools/detect_borders.py → tools/detect_borders.py
View File

0
ambience/image_tools/find_upscaled.py → tools/find_upscaled.py
View File

0
ambience/image_tools/find_watermarks.py → tools/find_watermarks.py
View File

0
tools/genertate_thumbnails.py Normal file
View File

0
ambience/image_tools/remove_borders.py → tools/remove_borders.py
View File

0
ambience/image_tools/yandex_rs.py → tools/yandex_rs.py
View File