@@ -118,15 +118,15 @@ export default function Image(props: any) {
export const getStaticProps: GetStaticProps = async (context) => {
if (context.params?.id) {
const img = await db_ops.image_ops.find_image_by_id(parseInt((context.params.id as string)))
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- let all_images_similaties:any= await fs.readFile("find_visually_similar_images/data.txt","utf-8")
- all_images_similaties=JSON.parse(all_images_similaties)
- let visually_similar_link=""
// console.log(all_images_similaties[(context.params.id as string)])
- if(all_images_similaties[(context.params.id as string)]!==undefined){
- visually_similar_link=`/visually_similar/${img[0].id}`
- }
if (img.length === 1) {
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ let all_images_similaties:any= await fs.readFile("find_visually_similar_images/data.txt","utf-8")
+ all_images_similaties=JSON.parse(all_images_similaties)
+ let visually_similar_link=""
+ if(all_images_similaties[(context.params.id as string)]!==undefined){
+ visually_similar_link=`/visually_similar/${img[0].id}`
+ }
const date = new Date(img[0].created_at)
const date_str = `${date.getDate()}.${date.getMonth() + 1}.${date.getFullYear()}`
const upscaled = (img[0].tags.includes('upscaled')?(`/upscaled/${img[0].id}.png`):null)
@@ -138,7 +138,8 @@ export const getStaticProps: GetStaticProps = async (context) => {
size: (img[0].size / (10 ** 6)).toFixed(2),
author: img[0].author,
tags: img[0].tags,
- derpi_link: img[0].derpi_link,
+ booru:img[0].booru,
+ booru_link: img[0].booru_link,
source_link: img[0].source_url,
date: date_str,
similar_by_tags_link:`/similar_by_tags/${img[0].id}`,
diff --git a/pages/import_from_derpi.tsx b/pages/import_from_derpi.tsx
index fff5b5f..97ba4c3 100644
--- a/pages/import_from_derpi.tsx
+++ b/pages/import_from_derpi.tsx
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
import AppBar from '../components/AppBar'
import db_ops from '../server/helpers/db_ops'
import Button from '@material-ui/core/Button';
@@ -6,6 +7,11 @@ import TextField from '@material-ui/core/TextField';
import Backdrop from '@material-ui/core/Backdrop';
import CircularProgress from '@material-ui/core/CircularProgress';
import { makeStyles } from '@material-ui/core/styles';
+import Radio from '@material-ui/core/Radio';
+import RadioGroup from '@material-ui/core/RadioGroup';
+import FormControlLabel from '@material-ui/core/FormControlLabel';
+import FormControl from '@material-ui/core/FormControl';
+import FormLabel from '@material-ui/core/FormLabel';
const useStyles = makeStyles(() => ({
backdrop: {
@@ -18,6 +24,10 @@ import { useState } from 'react';
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export default function Import_from_derpi(props: any) {
+ const [Booru, setBooru] = useState('');
+ const handleChange = (event:any) => {
+ setBooru(event.target.value);
+ };
if (props.err) {
return
}
@@ -30,22 +40,29 @@ export default function Import_from_derpi(props: any) {
}
};
const add_image = () => {
+ if(Booru===""){
+ alert("choose booru")
+ return
+ }
setOpen(true)
axios(`/import_from_derpi`, {
method: "post",
- data: { id: ImageID },
+ data: { id: ImageID,booru:Booru },
withCredentials: true,
timeout:5*60*1000
}).then((resp) => {
setOpen(false)
alert(JSON.stringify(resp.data))
setID(0)
+ setBooru("")
}).catch((err) => {
setOpen(false)
alert('check console for error message')
console.log(err)
setID(0)
+ setBooru("")
})
+
}
@@ -55,12 +72,20 @@ export default function Import_from_derpi(props: any) {
+
+ Booru
+
+ } label="derpibooru" />
+ } label="ponerpics" />
+ } label="ponybooru" />
+
+
setID(parseInt(e.target.value)||0)}
onKeyPress={(e) => handleKeyPress(e)}
diff --git a/server/bulk_import_images/bulk_import_images_from_derpi.ts b/server/bulk_import_images/bulk_import_images_from_derpi.ts
index 5223a4b..17b9c51 100644
--- a/server/bulk_import_images/bulk_import_images_from_derpi.ts
+++ b/server/bulk_import_images/bulk_import_images_from_derpi.ts
@@ -58,7 +58,7 @@ async function import_images() {
console.log(`imported ${image_file_name}`)
db_ops.image_ops.add_image(id, derpi_data.format.toLowerCase(), derpi_data.width, derpi_data.height, parsed_author, derpi_data.size,
derpi_link, derpi_data.upvotes, derpi_data.downvotes, derpi_data.id, derpi_data.created_at,
- derpi_data.source_url, derpi_data.tags, derpi_data.wilson_score, derpi_data.sha512_hash, phash,derpi_data.description)
+ derpi_data.source_url, derpi_data.tags, derpi_data.wilson_score, derpi_data.sha512_hash, phash,derpi_data.description,"derpibooru")
}
}
diff --git a/server/change_schema.ts b/server/change_schema.ts
new file mode 100644
index 0000000..4d492cd
--- /dev/null
+++ b/server/change_schema.ts
@@ -0,0 +1,29 @@
+import db_ops from './helpers/db_ops'
+
+async function change_schema() {
+ const images = await db_ops.image_ops.get_all_images();
+ for (const image of images) {
+ if(image.derpi_id){
+ const new_image=JSON.parse(JSON.stringify(image))
+ console.log(image.id)
+ new_image.booru="derpibooru"
+ new_image.booru_id=image.derpi_id
+ new_image.booru_likes=image.derpi_likes
+ new_image.booru_dislikes=image.derpi_dislikes
+ new_image.booru_link=image.derpi_link
+ new_image.booru_date=image.derpi_date
+ delete new_image.derpi_id
+ delete new_image.derpi_likes
+ delete new_image.derpi_dislikes
+ delete new_image.derpi_link
+ delete new_image.derpi_date
+ await db_ops.image_ops.delete_image_by_id(image.id)
+ await db_ops.image_ops.add_image_by_object(new_image)
+ }
+ }
+}
+async function run(){
+ await change_schema()
+ process.exit()
+}
+run()
diff --git a/server/helpers/db_ops.ts b/server/helpers/db_ops.ts
index 03451e0..064800b 100644
--- a/server/helpers/db_ops.ts
+++ b/server/helpers/db_ops.ts
@@ -47,12 +47,13 @@ async function findDocuments(collection_name:string, selector:Record) {
const collection = client.db(db_main).collection(collection_name);
- collection.deleteOne(selector)
+ return collection.deleteOne(selector)
+
}
async function insertDocuments(collection_name:string, documents:Array) {
const collection = client.db(db_main).collection(collection_name);
- collection.insertMany(documents)
+ return collection.insertMany(documents)
// const result = await collection.insertMany(documents);
// return result
}
@@ -212,9 +213,10 @@ async function find_image_by_id(id:number){
})
return img
}
-async function find_image_by_derpi_id(id:number){
+async function find_image_by_booru_id(booru:string,id:number){
const img = findDocuments("images", {
- derpi_id: id
+ booru:booru,
+ booru_id: id
})
return img
}
@@ -231,13 +233,16 @@ async function get_max_image_id(){
return result[0]?.id
}
async function delete_image_by_id(id:number){
- removeDocument("images",{id:id})
+ return removeDocument("images",{id:id})
+}
+async function add_image_by_object(image:any){
+ return insertDocuments("images", [image])
}
async function add_image(id:number,file_ext:string,width:number,height:number,author:string,
- size:string,derpi_link:string,
- derpi_likes:number,derpi_dislikes:number,
- derpi_id:number,derpi_date:Date,source_url:string,tags:Array,wilson_score:number,sha512:string,phash:string,description:string){
+ size:string,booru_link:string,
+ booru_likes:number,booru_dislikes:number,
+ booru_id:number,booru_date:Date,source_url:string,tags:Array,wilson_score:number,sha512:string,phash:string,description:string,booru:string){
insertDocuments("images", [{
id:id,
file_ext:file_ext,
@@ -250,11 +255,12 @@ async function add_image(id:number,file_ext:string,width:number,height:number,au
phash:phash,
sha512:sha512,
tags:tags,
- derpi_id:derpi_id,
- derpi_likes:derpi_likes,
- derpi_dislikes:derpi_dislikes,
- derpi_link:derpi_link,
- derpi_date:derpi_date,
+ booru:booru,
+ booru_id:booru_id,
+ booru_likes:booru_likes,
+ booru_dislikes:booru_dislikes,
+ booru_link:booru_link,
+ booru_date:booru_date,
source_url:source_url,
wilson_score:wilson_score
}])
@@ -379,6 +385,7 @@ async function create_new_user_not_activated(email:string, pass:string, token:st
export default {
image_ops: {
add_image,
+ add_image_by_object,
get_all_images,
find_image_by_id,
get_max_image_id,
@@ -387,7 +394,7 @@ export default {
get_ids_and_phashes,
find_image_by_phash,
find_image_by_sha512,
- find_image_by_derpi_id,
+ find_image_by_booru_id,
update_image_data_by_id,
add_tags_to_image_by_id
},
diff --git a/server/routes/import_from_derpi.ts b/server/routes/import_from_derpi.ts
index 2044e22..a99cf26 100644
--- a/server/routes/import_from_derpi.ts
+++ b/server/routes/import_from_derpi.ts
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
/* eslint-disable @typescript-eslint/no-var-requires */
import db_ops from './../helpers/db_ops'
@@ -18,44 +19,56 @@ async function parse_author(tags: any) {
}
return "???"
}
-
+function get_domain(booru:string){
+ switch(booru){
+ case "derpibooru":
+ return "https://www.derpibooru.org"
+ case "ponerpics":
+ return "https://ponerpics.org"
+ case "ponybooru":
+ return "https://ponybooru.org"
+ }
+}
async function import_from_derpi(req: Request, res: Response) {
- const derpy_import_id = parseInt(req.body.id);
+ const allowed_boorus=["derpibooru","ponerpics","ponybooru"]
+ const booru_import_id = parseInt(req.body.id);
const ALLOWED_FORMATS = ["png", 'jpg', "jpeg"]
- if (req.session?.user_id) {
+ const booru=req.body.booru
+ if (req.session?.user_id && !isNaN(booru_import_id) && allowed_boorus.includes(booru)) {
const user = await db_ops.activated_user.find_user_by_id(req.session?.user_id)
if (user[0].isAdmin) {
req.setTimeout(5*60*1000)
try {
- const imgs = await db_ops.image_ops.find_image_by_derpi_id(derpy_import_id)
+ const booru_domain=get_domain(req.body.booru)
+ const imgs = await db_ops.image_ops.find_image_by_booru_id(booru,booru_import_id)
if (imgs.length !== 0) {
res.json({ message: "Already in the DB" })
return
}
- const response = await axios.get(`https://www.derpibooru.org/api/v1/json/images/${derpy_import_id}`,{ headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36' } });
- const derpi_data = response.data.image
- if (!ALLOWED_FORMATS.includes(derpi_data.format.toLowerCase())) {
+ const response = await axios.get(`${booru_domain}/api/v1/json/images/${booru_import_id}`,{ headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36' } });
+ const booru_image_data = response.data.image
+ if (!ALLOWED_FORMATS.includes(booru_image_data.format.toLowerCase())) {
res.json({ message: "format is not in allowed formats" })
return
}
- const image =await axios.get(derpi_data.representations.full, {responseType: 'arraybuffer'})
+ const image =await axios.get((booru==="ponerpics")?booru_domain+booru_image_data.representations.full:booru_image_data.representations.full, {responseType: 'arraybuffer'})
const new_image_id = (await db_ops.image_ops.get_max_image_id())+1
await thumbnail_ops.generate_thumbnail(image.data,new_image_id)
- fs.writeFile(`${PATH_TO_IMAGES}/${new_image_id}.${derpi_data.format.toLowerCase()}`, image.data, 'binary', function (err) {
+ fs.writeFile(`${PATH_TO_IMAGES}/${new_image_id}.${booru_image_data.format.toLowerCase()}`, image.data, 'binary', function (err) {
if (err) {
- console.log(`There was an error writing the image: derpi_id: ${derpy_import_id} id: ${new_image_id}`)
+ console.log(`There was an error writing the image: ${booru} booru_id: ${booru_import_id} id: ${new_image_id}`)
}
});
- const parsed_author = await parse_author(derpi_data.tags)
- const derpi_link = "https://derpibooru.org/images/" + derpi_data.id
+ const parsed_author = await parse_author(booru_image_data.tags)
+ const booru_link = `${booru_domain}/images/${booru_image_data.id}`
const phash = await imghash.hash(image.data, 16);
await image_ops.calculate_sift_features(new_image_id,image.data)
await image_ops.calculate_color_hist_and_similarities(new_image_id,image.data)
- derpi_data.tags.push(`width:${derpi_data.width}`)
- derpi_data.tags.push(`height:${derpi_data.height}`)
- await db_ops.image_ops.add_image(new_image_id, derpi_data.format.toLowerCase(), derpi_data.width, derpi_data.height, parsed_author, derpi_data.size,
- derpi_link, derpi_data.upvotes, derpi_data.downvotes, derpi_data.id, derpi_data.created_at,
- derpi_data.source_url, derpi_data.tags, derpi_data.wilson_score, derpi_data.sha512_hash, phash, derpi_data.description)
+ booru_image_data.tags.push(`width:${booru_image_data.width}`)
+ booru_image_data.tags.push(`height:${booru_image_data.height}`)
+ await db_ops.image_ops.add_image(new_image_id, booru_image_data.format.toLowerCase(), booru_image_data.width, booru_image_data.height, parsed_author, booru_image_data.size,
+ booru_link, booru_image_data.upvotes, booru_image_data.downvotes, booru_image_data.id, booru_image_data.created_at,
+ booru_image_data.source_url, booru_image_data.tags, booru_image_data.wilson_score, booru_image_data.sha512_hash, phash, booru_image_data.description,booru)
console.log(`OK. New image_id: ${new_image_id}`)
res.json({ message: `OK. New image_id: ${new_image_id}`})
return