REVERSE SEARCH 2 ORB EDITION

pages/reverse_search.tsx

@@ -22,30 +22,33 @@ export default function ReverseSearch() {
   const router = useRouter()
   const [Files, setFiles] = useState([]);
   const [open, setOpen] = useState(false);
-  const send_image = (token: string) => {
+  const send_image = (token: string,mode:string) => {
     setOpen(true)
     const formData = new FormData();
     formData.append("image", Files[0]);
     formData.append("g-recaptcha-response", token);
+    formData.append("mode", mode);
     axios(`/reverse_search`, {
       method: "post",
       data: formData,
       headers: {
         'Content-Type': 'multipart/form-data'
-      }
+      },
+      timeout:120000   //2min
     }).then((resp) => {
       setOpen(false)
+      console.log(resp.data.ids)
       router.push("/show?ids=" + resp.data.ids)
     }).catch((err) => {
       setOpen(false)
       console.log(err)
     })
   }
-  const _send_image = () => {
+  const _send_image = (mode:string) => {
     /*global grecaptcha*/ // defined in pages/_document.tsx
     grecaptcha.ready(function () {
       grecaptcha.execute(config.recaptcha_site_key, { action: 'login' }).then(function (token) {
-        send_image(token)
+        send_image(token,mode)
       });
     })
   }
@@ -64,7 +67,9 @@ export default function ReverseSearch() {
           onChange={(files) => setFiles((files as never))}
         />
       </Box>
-      <Button onClick={() => { _send_image() }} variant="contained" color="primary" >Reverse Search</Button>
+      <Button onClick={() => { _send_image("1") }} variant="contained" color="primary" >Reverse Search (fast, less accureate)</Button>
+      <div style={{marginTop:"10px"}}><Button onClick={() => { _send_image("2") }} variant="contained" color="primary" >Reverse Search (slow, more accurate)</Button></div>
+      
     </div>
   );
 }

server/bulk_calculate_orb_features/calculate_orb_features.ts

@@ -0,0 +1,36 @@
+import * as cv from 'opencv4nodejs'
+import path from 'path';
+import db_ops from '../helpers/db_ops'
+import config from '../../config/config'
+const detector=new cv.ORBDetector()
+const PATH_TO_IMAGES = path.join(config.root_path, 'public', 'images')
+
+async function calculate_orb_features(){
+    const images = await db_ops.image_ops.get_all_images()
+    for(const image of images){
+        const check_if_already_calculated= await db_ops.image_search.get_orb_features_by_id(image.id)
+        if(check_if_already_calculated.length!==0){
+          continue
+        }
+        console.log(image.id)
+        try{
+          const img = await cv.imreadAsync(`${PATH_TO_IMAGES}/${image.id}.${image.file_ext}`);
+          const keyPoints = await detector.detectAsync(img);
+          const descriptors = await detector.computeAsync(img, keyPoints);
+          const descriptors_as_array=descriptors.getDataAsArray()
+          descriptors.release()
+          img.release()
+          await db_ops.image_search.add_orb_features_by_id(image.id,descriptors_as_array)
+        }catch(err){
+          console.log(err)
+          console.log(image.id)
+        }
+     
+    }
+}
+
+async function run() {
+  await calculate_orb_features()
+  process.exit()
+}
+run()

server/bulk_calculate_orb_features/delete_orb_features.ts

@@ -0,0 +1,11 @@
+import db_ops from '../helpers/db_ops'
+
+async function delete_orb_feature_by_id() {
+//  const color_hist_image_ids=[]
+ const similarities_image_ids=[-1]
+ for (const id of similarities_image_ids){
+   await db_ops.image_search.delete_orb_feature_by_id(id)
+ }
+ process.exit()
+}
+delete_orb_feature_by_id()

server/bulk_calculate_sift_features/calculate_sift_features.ts

@@ -1,29 +0,0 @@
-import * as cv from 'opencv4nodejs'
-import path from 'path';
-import db_ops from '../helpers/db_ops'
-import config from '../../config/config'
-const detector=new cv.SIFTDetector({ nFeatures: 400 })
-const PATH_TO_IMAGES = path.join(config.root_path, 'public', 'images')
-
-async function calculate_color_hist(){
-    const images = await db_ops.image_ops.get_all_images()
-    for(const image of images){
-        const check_if_already_calculated= await db_ops.image_search.get_sift_features_by_id(image.id)
-        if(check_if_already_calculated.length!==0){
-          continue
-        }
-        console.log(image.id)
-        const img = await cv.imreadAsync(`${PATH_TO_IMAGES}/${image.id}.${image.file_ext}`);
-        const keyPoints = await detector.detectAsync(img);
-        const descriptors = await detector.computeAsync(img, keyPoints);
-        descriptors.release()
-        img.release()
-        const descriptors_as_array=descriptors.getDataAsArray()
-        await db_ops.image_search.add_sift_features_by_id(image.id,descriptors_as_array)
-    }
-}
-
-async function run() {
-  await calculate_color_hist()
-}
-run()

server/helpers/db_ops.ts

@@ -91,6 +91,36 @@ async function generate_id() {
 }
 /////////////////////////////////////////////////IMAGE SEARCH OPS
 
+async function delete_orb_feature_by_id(id:number){
+    removeDocument("orb_reverse_search",{id:id})
+}
+
+async function add_orb_features_by_id(id:number,orb_features:Array<any>){
+    insertDocuments("orb_reverse_search", [{
+        id:id,
+        orb_features:orb_features
+    }])
+}
+
+async function get_orb_features_batch(skip:number,limit:number){
+    const collection = client.db(db_main).collection("orb_reverse_search");
+    const similarities = collection.find().skip(skip).limit(limit).project({_id:0}).toArray()
+    return similarities
+}
+
+
+async function get_orb_features_by_id(id:number){
+    const collection = client.db(db_main).collection("orb_reverse_search");
+    const similarities = collection.find({id:id}).project({_id:0,id:0}).toArray()
+    return similarities
+}
+async function get_number_of_images_orb_reverse_search(){
+    const collection = client.db(db_main).collection("orb_reverse_search");
+    const number_of_images_in_collection = collection.countDocuments()
+    return number_of_images_in_collection
+}
+
+
 async function get_color_similarities_by_id(id:number){
 const collection = client.db(db_main).collection("color_similarities");
 // collection.find(selector).project({_id:0}).explain((_err,exp)=>console.log(exp))
@@ -364,6 +394,11 @@ export default {
         add_tags_to_image_by_id
     },
     image_search:{
+        delete_orb_feature_by_id,
+        add_orb_features_by_id,
+        get_orb_features_batch,
+        get_orb_features_by_id,
+        get_number_of_images_orb_reverse_search,
         get_all_color_hists,
         get_color_hist_by_id,
         add_color_hist_by_id,

server/helpers/image_ops.ts

@@ -1,6 +1,12 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
 import * as cv from 'opencv4nodejs'
 import { HistAxes } from 'opencv4nodejs';
 import db_ops from './db_ops';
+
+const detector=new cv.ORBDetector()
+const matchFunc=cv.matchBruteForceHammingAsync
+const imghash = require('imghash');
+
 const BIN_SIZE=16
 const histAxes:HistAxes[]= [
     new HistAxes({
@@ -19,6 +25,7 @@ const histAxes:HistAxes[]= [
         ranges: [0, 255]
       }),
   ]
+
   async function calculate_color_hist_and_similarities(new_image_id:number,image:Buffer){
         const img_mat = await cv.imdecodeAsync(image)
         let rgb_hist = await cv.calcHistAsync(img_mat, histAxes)
@@ -39,4 +46,58 @@ const histAxes:HistAxes[]= [
         }
         await db_ops.image_search.add_color_similarities_by_id(new_image_id,similarities)
 }
-export default {calculate_color_hist_and_similarities}
+async function get_similar_images_by_orb(image:Buffer) {
+  const img_mat = await cv.imdecodeAsync(image)
+  const keyPoints = await detector.detectAsync(img_mat);
+  const img_descriptors = await detector.computeAsync(img_mat, keyPoints);
+  const number_of_images = await db_ops.image_search.get_number_of_images_orb_reverse_search()
+  const batch = 500;
+  const similar_images=[]
+  console.time()
+  for (let i = 0; i < number_of_images; i += batch) {
+    const descriptors = await db_ops.image_search.get_orb_features_batch(i, batch) 
+    for (const img of descriptors) {
+      const descriptors2 = new cv.Mat(img.orb_features, cv.CV_8UC1)
+      const matches = await matchFunc(img_descriptors, descriptors2);
+      descriptors2.release()
+      let sum = 0
+      for (const x of matches) {
+        sum += x.distance
+      }
+      if (sum===0){
+        return [img.id]
+      }
+      similar_images.push({id:img.id,avg_distance:sum / matches.length})
+    }
+  }
+  console.timeEnd()
+  similar_images.sort((a,b)=>a.avg_distance-b.avg_distance)
+  similar_images.length=30
+  const ids=similar_images.map((el)=>el.id)
+  return ids
+}
+function hamming_distance(str1: string, str2: string) {
+  let distance = 0;
+  for (let i = 0; i < str1.length; i += 1) {
+      if (str1[i] !== str2[i]) {
+          distance += 1;
+      }
+  }
+  return distance;
+}
+
+async function get_similar_images_by_phash(image:Buffer){
+  const phash= await imghash.hash(image,16)
+  const images=await db_ops.image_ops.get_ids_and_phashes()
+  for(let i=0;i<images.length;i++){
+      images[i].dist=hamming_distance(phash,images[i].phash)
+      if(images[i].dist===0){
+        return [images[i].id]
+      }
+  }
+  images.sort((a,b)=>a.dist-b.dist)
+  images.length=30
+  const ids=images.map((el)=>el.id)
+  return ids
+}
+export default {calculate_color_hist_and_similarities,get_similar_images_by_orb,get_similar_images_by_phash}

server/routes/reverse_search.ts

@@ -3,17 +3,8 @@
 // import db_ops from './../helpers/db_ops'
 import { Request, Response } from 'express';
 import { RecaptchaResponseV3 } from 'express-recaptcha/dist/interfaces';
-import db_ops from '../helpers/db_ops'
-const imghash: any = require('imghash');
-function hamming_distance(str1: string, str2: string) {
-    let distance = 0;
-    for (let i = 0; i < str1.length; i += 1) {
-        if (str1[i] !== str2[i]) {
-            distance += 1;
-        }
-    }
-    return distance;
-}
+import image_ops from '../helpers/image_ops'
+
 async function reverse_search(req: Request, res: Response) {
     const recaptcha_score=(req.recaptcha as RecaptchaResponseV3)?.data?.score
     if (req.recaptcha?.error|| (typeof recaptcha_score==="number" && recaptcha_score<0.5)) {
@@ -21,15 +12,21 @@ async function reverse_search(req: Request, res: Response) {
             message: "Captcha error"
         });
     }
-    const phash= await imghash.hash(req.file.buffer,16)
-    const images=await db_ops.image_ops.get_ids_and_phashes()
-    for(let i=0;i<images.length;i++){
-        images[i].dist=hamming_distance(phash,images[i].phash)
+    
+    if(req.file){
+        const Mode=parseInt(req.body.mode)
+        req.setTimeout(120000)//2min
+        if(Mode===1){
+            const ids=await image_ops.get_similar_images_by_phash(req.file.buffer)
+            console.log(ids)
+            res.json({ids:ids.join(',')})
+        }else if(Mode===2){
+            const ids=await image_ops.get_similar_images_by_orb(req.file.buffer)
+            console.log(ids)
+            res.json({ids:ids.join(',')})
+        }
+       
     }
-    images.sort((a,b)=>a.dist-b.dist)
-    images.length=30
-    const ids=images.map((el)=>el.id)
-    res.json({ids:ids.join(',')})
 }
 
 export default reverse_search;