USEFULL/qw-gallery-scenery
1
0
Fork 0
mirror of https://github.com/qwertyforce/scenery.git synced 2025-05-02 05:30:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

import from other boorus

Browse Source
This commit is contained in:
qwertyforce 2020-12-21 22:27:57 +03:00
parent 553d31ade3
commit edd80d44e2
6 changed files with 119 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
pages/image/[id].tsx
View File

@ -77,7 +77,7 @@ export default function Image(props: any) {
</div>
<div className={classes.icon_container}>
<LinkIcon />
&nbsp;<a href={props.derpi_link} target="_blank" rel="noreferrer">Derpi Link</a>
&nbsp;<a href={props.booru_link} target="_blank" rel="noreferrer">{props.booru} link</a>
</div>
<div className={classes.icon_container}>
<LinkIcon />
@ -118,15 +118,15 @@ export default function Image(props: any) {
export const getStaticProps: GetStaticProps = async (context) => {
if (context.params?.id) {
const img = await db_ops.image_ops.find_image_by_id(parseInt((context.params.id as string)))
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let all_images_similaties:any= await fs.readFile("find_visually_similar_images/data.txt","utf-8")
all_images_similaties=JSON.parse(all_images_similaties)
let visually_similar_link=""
// console.log(all_images_similaties[(context.params.id as string)])
if(all_images_similaties[(context.params.id as string)]!==undefined){
visually_similar_link=`/visually_similar/${img[0].id}`
}
if (img.length === 1) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let all_images_similaties:any= await fs.readFile("find_visually_similar_images/data.txt","utf-8")
all_images_similaties=JSON.parse(all_images_similaties)
let visually_similar_link=""
if(all_images_similaties[(context.params.id as string)]!==undefined){
visually_similar_link=`/visually_similar/${img[0].id}`
}
const date = new Date(img[0].created_at)
const date_str = `${date.getDate()}.${date.getMonth() + 1}.${date.getFullYear()}`
const upscaled = (img[0].tags.includes('upscaled')?(`/upscaled/${img[0].id}.png`):null)
@ -138,7 +138,8 @@ export const getStaticProps: GetStaticProps = async (context) => {
size: (img[0].size / (10 ** 6)).toFixed(2),
author: img[0].author,
tags: img[0].tags,
derpi_link: img[0].derpi_link,
booru:img[0].booru,
booru_link: img[0].booru_link,
source_link: img[0].source_url,
date: date_str,
similar_by_tags_link:`/similar_by_tags/${img[0].id}`,

31
pages/import_from_derpi.tsx
View File

@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import AppBar from '../components/AppBar'
import db_ops from '../server/helpers/db_ops'
import Button from '@material-ui/core/Button';
@ -6,6 +7,11 @@ import TextField from '@material-ui/core/TextField';
import Backdrop from '@material-ui/core/Backdrop';
import CircularProgress from '@material-ui/core/CircularProgress';
import { makeStyles } from '@material-ui/core/styles';
import Radio from '@material-ui/core/Radio';
import RadioGroup from '@material-ui/core/RadioGroup';
import FormControlLabel from '@material-ui/core/FormControlLabel';
import FormControl from '@material-ui/core/FormControl';
import FormLabel from '@material-ui/core/FormLabel';
const useStyles = makeStyles(() => ({
backdrop: {
@ -18,6 +24,10 @@ import { useState } from 'react';
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export default function Import_from_derpi(props: any) {
const [Booru, setBooru] = useState('');
const handleChange = (event:any) => {
setBooru(event.target.value);
};
if (props.err) {
return <ErrorPage statusCode={404} />
}
@ -30,22 +40,29 @@ export default function Import_from_derpi(props: any) {
}
};
const add_image = () => {
if(Booru===""){
alert("choose booru")
return
}
setOpen(true)
axios(`/import_from_derpi`, {
method: "post",
data: { id: ImageID },
data: { id: ImageID,booru:Booru },
withCredentials: true,
timeout:5*60*1000
}).then((resp) => {
setOpen(false)
alert(JSON.stringify(resp.data))
setID(0)
setBooru("")
}).catch((err) => {
setOpen(false)
alert('check console for error message')
console.log(err)
setID(0)
setBooru("")
})
}
@ -55,12 +72,20 @@ export default function Import_from_derpi(props: any) {
<Backdrop className={classes.backdrop} open={open}>
<CircularProgress color="inherit" />
</Backdrop>
<FormControl component="fieldset">
<FormLabel component="legend">Booru</FormLabel>
<RadioGroup aria-label="booru" name="booru" value={Booru} onChange={handleChange}>
<FormControlLabel value="derpibooru" control={<Radio />} label="derpibooru" />
<FormControlLabel value="ponerpics" control={<Radio />} label="ponerpics" />
<FormControlLabel value="ponybooru" control={<Radio />} label="ponybooru" />
</RadioGroup>
</FormControl>
<TextField
value ={ImageID}
fullWidth
type="number"
label="Derpi image id"
placeholder="Derpi image id"
label="image id"
placeholder="image id"
margin="normal"
onChange={(e) => setID(parseInt(e.target.value)||0)}
onKeyPress={(e) => handleKeyPress(e)}

2
server/bulk_import_images/bulk_import_images_from_derpi.ts
View File

@ -58,7 +58,7 @@ async function import_images() {
console.log(`imported ${image_file_name}`)
db_ops.image_ops.add_image(id, derpi_data.format.toLowerCase(), derpi_data.width, derpi_data.height, parsed_author, derpi_data.size,
derpi_link, derpi_data.upvotes, derpi_data.downvotes, derpi_data.id, derpi_data.created_at,
derpi_data.source_url, derpi_data.tags, derpi_data.wilson_score, derpi_data.sha512_hash, phash,derpi_data.description)
derpi_data.source_url, derpi_data.tags, derpi_data.wilson_score, derpi_data.sha512_hash, phash,derpi_data.description,"derpibooru")
}
}

29
server/change_schema.ts Normal file
View File

@ -0,0 +1,29 @@
import db_ops from './helpers/db_ops'
async function change_schema() {
const images = await db_ops.image_ops.get_all_images();
for (const image of images) {
if(image.derpi_id){
const new_image=JSON.parse(JSON.stringify(image))
console.log(image.id)
new_image.booru="derpibooru"
new_image.booru_id=image.derpi_id
new_image.booru_likes=image.derpi_likes
new_image.booru_dislikes=image.derpi_dislikes
new_image.booru_link=image.derpi_link
new_image.booru_date=image.derpi_date
delete new_image.derpi_id
delete new_image.derpi_likes
delete new_image.derpi_dislikes
delete new_image.derpi_link
delete new_image.derpi_date
await db_ops.image_ops.delete_image_by_id(image.id)
await db_ops.image_ops.add_image_by_object(new_image)
}
}
}
async function run(){
await change_schema()
process.exit()
}
run()

35
server/helpers/db_ops.ts
View File

@ -47,12 +47,13 @@ async function findDocuments(collection_name:string, selector:Record<string,unkn
}
async function removeDocument(collection_name:string, selector:Record<string,unknown>) {
const collection = client.db(db_main).collection(collection_name);
collection.deleteOne(selector)
return collection.deleteOne(selector)
}
async function insertDocuments(collection_name:string, documents:Array<any>) {
const collection = client.db(db_main).collection(collection_name);
collection.insertMany(documents)
return collection.insertMany(documents)
// const result = await collection.insertMany(documents);
// return result
}
@ -212,9 +213,10 @@ async function find_image_by_id(id:number){
})
return img
}
async function find_image_by_derpi_id(id:number){
async function find_image_by_booru_id(booru:string,id:number){
const img = findDocuments("images", {
derpi_id: id
booru:booru,
booru_id: id
})
return img
}
@ -231,13 +233,16 @@ async function get_max_image_id(){
return result[0]?.id
}
async function delete_image_by_id(id:number){
removeDocument("images",{id:id})
return removeDocument("images",{id:id})
}
async function add_image_by_object(image:any){
return insertDocuments("images", [image])
}
async function add_image(id:number,file_ext:string,width:number,height:number,author:string,
size:string,derpi_link:string,
derpi_likes:number,derpi_dislikes:number,
derpi_id:number,derpi_date:Date,source_url:string,tags:Array<string>,wilson_score:number,sha512:string,phash:string,description:string){
size:string,booru_link:string,
booru_likes:number,booru_dislikes:number,
booru_id:number,booru_date:Date,source_url:string,tags:Array<string>,wilson_score:number,sha512:string,phash:string,description:string,booru:string){
insertDocuments("images", [{
id:id,
file_ext:file_ext,
@ -250,11 +255,12 @@ async function add_image(id:number,file_ext:string,width:number,height:number,au
phash:phash,
sha512:sha512,
tags:tags,
derpi_id:derpi_id,
derpi_likes:derpi_likes,
derpi_dislikes:derpi_dislikes,
derpi_link:derpi_link,
derpi_date:derpi_date,
booru:booru,
booru_id:booru_id,
booru_likes:booru_likes,
booru_dislikes:booru_dislikes,
booru_link:booru_link,
booru_date:booru_date,
source_url:source_url,
wilson_score:wilson_score
}])
@ -379,6 +385,7 @@ async function create_new_user_not_activated(email:string, pass:string, token:st
export default {
image_ops: {
add_image,
add_image_by_object,
get_all_images,
find_image_by_id,
get_max_image_id,
@ -387,7 +394,7 @@ export default {
get_ids_and_phashes,
find_image_by_phash,
find_image_by_sha512,
find_image_by_derpi_id,
find_image_by_booru_id,
update_image_data_by_id,
add_tags_to_image_by_id
},

47
server/routes/import_from_derpi.ts
View File

@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
/* eslint-disable @typescript-eslint/no-var-requires */
import db_ops from './../helpers/db_ops'
@ -18,44 +19,56 @@ async function parse_author(tags: any) {
}
return "???"
}
function get_domain(booru:string){
switch(booru){
case "derpibooru":
return "https://www.derpibooru.org"
case "ponerpics":
return "https://ponerpics.org"
case "ponybooru":
return "https://ponybooru.org"
}
}
async function import_from_derpi(req: Request, res: Response) {
const derpy_import_id = parseInt(req.body.id);
const allowed_boorus=["derpibooru","ponerpics","ponybooru"]
const booru_import_id = parseInt(req.body.id);
const ALLOWED_FORMATS = ["png", 'jpg', "jpeg"]
if (req.session?.user_id) {
const booru=req.body.booru
if (req.session?.user_id && !isNaN(booru_import_id) && allowed_boorus.includes(booru)) {
const user = await db_ops.activated_user.find_user_by_id(req.session?.user_id)
if (user[0].isAdmin) {
req.setTimeout(5*60*1000)
try {
const imgs = await db_ops.image_ops.find_image_by_derpi_id(derpy_import_id)
const booru_domain=get_domain(req.body.booru)
const imgs = await db_ops.image_ops.find_image_by_booru_id(booru,booru_import_id)
if (imgs.length !== 0) {
res.json({ message: "Already in the DB" })
return
}
const response = await axios.get(`https://www.derpibooru.org/api/v1/json/images/${derpy_import_id}`,{ headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36' } });
const derpi_data = response.data.image
if (!ALLOWED_FORMATS.includes(derpi_data.format.toLowerCase())) {
const response = await axios.get(`${booru_domain}/api/v1/json/images/${booru_import_id}`,{ headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36' } });
const booru_image_data = response.data.image
if (!ALLOWED_FORMATS.includes(booru_image_data.format.toLowerCase())) {
res.json({ message: "format is not in allowed formats" })
return
}
const image =await axios.get(derpi_data.representations.full, {responseType: 'arraybuffer'})
const image =await axios.get((booru==="ponerpics")?booru_domain+booru_image_data.representations.full:booru_image_data.representations.full, {responseType: 'arraybuffer'})
const new_image_id = (await db_ops.image_ops.get_max_image_id())+1
await thumbnail_ops.generate_thumbnail(image.data,new_image_id)
fs.writeFile(`${PATH_TO_IMAGES}/${new_image_id}.${derpi_data.format.toLowerCase()}`, image.data, 'binary', function (err) {
fs.writeFile(`${PATH_TO_IMAGES}/${new_image_id}.${booru_image_data.format.toLowerCase()}`, image.data, 'binary', function (err) {
if (err) {
console.log(`There was an error writing the image: derpi_id: ${derpy_import_id} id: ${new_image_id}`)
console.log(`There was an error writing the image: ${booru} booru_id: ${booru_import_id} id: ${new_image_id}`)
}
});
const parsed_author = await parse_author(derpi_data.tags)
const derpi_link = "https://derpibooru.org/images/" + derpi_data.id
const parsed_author = await parse_author(booru_image_data.tags)
const booru_link = `${booru_domain}/images/${booru_image_data.id}`
const phash = await imghash.hash(image.data, 16);
await image_ops.calculate_sift_features(new_image_id,image.data)
await image_ops.calculate_color_hist_and_similarities(new_image_id,image.data)
derpi_data.tags.push(`width:${derpi_data.width}`)
derpi_data.tags.push(`height:${derpi_data.height}`)
await db_ops.image_ops.add_image(new_image_id, derpi_data.format.toLowerCase(), derpi_data.width, derpi_data.height, parsed_author, derpi_data.size,
derpi_link, derpi_data.upvotes, derpi_data.downvotes, derpi_data.id, derpi_data.created_at,
derpi_data.source_url, derpi_data.tags, derpi_data.wilson_score, derpi_data.sha512_hash, phash, derpi_data.description)
booru_image_data.tags.push(`width:${booru_image_data.width}`)
booru_image_data.tags.push(`height:${booru_image_data.height}`)
await db_ops.image_ops.add_image(new_image_id, booru_image_data.format.toLowerCase(), booru_image_data.width, booru_image_data.height, parsed_author, booru_image_data.size,
booru_link, booru_image_data.upvotes, booru_image_data.downvotes, booru_image_data.id, booru_image_data.created_at,
booru_image_data.source_url, booru_image_data.tags, booru_image_data.wilson_score, booru_image_data.sha512_hash, phash, booru_image_data.description,booru)
console.log(`OK. New image_id: ${new_image_id}`)
res.json({ message: `OK. New image_id: ${new_image_id}`})
return