From 65af1305221b958e60fdd5f47630b7e0af7e8106 Mon Sep 17 00:00:00 2001 From: listofbanned Date: Wed, 19 Oct 2022 17:21:38 -0300 Subject: [PATCH 1/8] add page to test generators/deploment endpoints --- constants/endpoints-constants.js | 12 +- generators/generator.js | 171 +++++++++++++++++++++ pages/generators.js | 246 +++++++++++++++++++++++++++++++ 3 files changed, 426 insertions(+), 3 deletions(-) create mode 100644 generators/generator.js create mode 100644 pages/generators.js diff --git a/constants/endpoints-constants.js b/constants/endpoints-constants.js index 569bb92..689640f 100644 --- a/constants/endpoints-constants.js +++ b/constants/endpoints-constants.js @@ -1,4 +1,10 @@ -export const weaviateUrl = `https://weaviate.weabaverse.com`; +export const gpt3Url = `https://gpt3.webaverse.com`; // check | create +export const voiceUrl = `https://voice-cw.webaverse.com`; // test export const stableDiffusionUrl = `https://stable-diffusion.webaverse.com`; -export const voiceUrl = `https://voice-cw.webaverse.com`; -export const diffsoundUrl = `https://diffsound.webaverse.com`; \ No newline at end of file +export const diffsoundUrl = `https://diffsound.webaverse.com`; // test +export const motionDiffusionUrl = `https://motion-diffusion.webaverse.com`; // check | create +export const stableDreamfusionUrl = `https://stable-dreamfussion.webaverse.com`; // check | create +export const get3dUrl = `https://get-3d.webaverse.com`; // check | create +export const musicGeneratorUrl = `https://music-generator.webaverse.com`; // check | create +export const weaviateUrl = `https://weaviate.weabaverse.com`; // test +export const discoDiffusionUrl = `https://disco-diffusion.weabaverse.com`; // check | create \ No newline at end of file diff --git a/generators/generator.js b/generators/generator.js new file mode 100644 index 0000000..2c3ed24 --- /dev/null +++ b/generators/generator.js @@ -0,0 +1,171 @@ +// import {stableDiffusionUrl} from '../../constants/endpoints.js'; +import fetch from 'node-fetch'; +import { + gpt3Url, + voiceUrl, + stableDiffusionUrl, + diffsoundUrl, + motionDiffusionUrl, + stableDreamfusionUrl, + get3dUrl, + musicGeneratorUrl, + weaviateUrl, + discoDiffusionUrl, +} from '../constants/endpoints-constants'; + +export const generateText = ({model}) => async ({x} = {}) => { + const url = `${gpt3Url}/text?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated text + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateVoice = ({model}) => async ({x} = {}) => { + const url = `${voiceUrl}/voice?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated voice + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateImage = ({ + modelName, + prefix, +}) => async ({ + name, + description, +} = {}) => { + const s = `${prefix} ${description}`; + const u = `${stableDiffusionUrl}/image?s=${encodeURIComponent(s)}&model=${modelName}`; + const res = await fetch(u); + if (res.ok) { + const arrayBuffer = await res.arrayBuffer(); + if (arrayBuffer.byteLength > 0) { + return arrayBuffer; + } else { + throw new Error(`generated empty image`); + } + } else { + throw new Error(`invalid status: ${res.status}`); + } +} + +export const generateDiffSound = ({model}) => async ({x} = {}) => { + const url = `${diffsoundUrl}/sound?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated sound + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateMotionDiffusion = ({model}) => async ({x} = {}) => { + const url = `${motionDiffusionUrl}/motion?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated motion + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateObjectOrConsumable = ({model}) => async ({x} = {}) => { + const url = `${stableDreamfusionUrl}/object?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated object | consumable + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateGet3DObject = ({model}) => async ({x} = {}) => { + const url = `${get3dUrl}/object?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated object + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateMusic = ({model}) => async ({x} = {}) => { + const url = `${musicGeneratorUrl}/music?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated music + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateWeaviateCharacter = ({model}) => async ({x} = {}) => { + const url = `${weaviateUrl}/character?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated character ?? + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} + +export const generateSprite = ({model}) => async ({x} = {}) => { + const url = `${discoDiffusionUrl}/sprite?x=${x}` // mock endpoint + await fetch(url) + .then(res => { + if (res.ok) { + // TODO: return generated sprite + } else { + throw new Error(`invalid status: ${res.status}`); + } + }) + .catch(err => { + throw new Error(`url error: ${err}`); + }) +} diff --git a/pages/generators.js b/pages/generators.js new file mode 100644 index 0000000..5faddf6 --- /dev/null +++ b/pages/generators.js @@ -0,0 +1,246 @@ +import Head from 'next/head' +import {useState} from 'react' +import styles from '../styles/Home.module.css' +import { + generateText, + generateVoice, + generateImage, + generateDiffSound, + generateMotionDiffusion, + generateObjectOrConsumable, + generateGet3DObject, + generateMusic, + generateWeaviateCharacter, + generateSprite, +} from '../generators/generator' + +export default function Generators() { + const [loadingText, setLoadingText] = useState(false) + const [generatedText, setGeneratedText] = useState(null) + + const [loadingVoice, setLoadingVoice] = useState(false) + const [generatedVoice, setGeneratedVoice] = useState(null) + + const [loadingImage, setLoadingImage] = useState(false) + const [generatedImage, setGeneratedImage] = useState(null) + + const [loadingSound, setLoadingSound] = useState(false) + const [generatedSound, setGeneratedSound] = useState(null) + + const [loadingMotion, setLoadingMotion] = useState(false) + const [generatedMotion, setGeneratedMotion] = useState(null) + + const [loadingObject, setLoadingObject] = useState(false) + const [generatedObject, setGeneratedObject] = useState(null) + + const [loadingGet3DObject, setLoadingGet3DObject] = useState(false) + const [generatedGet3DObject, setGeneratedGet3DObject] = useState(null) + + const [loadingMusic, setLoadingMusic] = useState(false) + const [generatedMusic, setGeneratedMusic] = useState(null) + + const [loadingCharacter, setLoadingCharacter] = useState(false) + const [generatedCharacter, setGeneratedCharacter] = useState(null) + + const [loadingSprite, setLoadingSprite] = useState(false) + const [generatedSprite, setGeneratedSprite] = useState(null) + + // generateText + async function generateTestText() { + setLoadingText(true) + // TODO generateText() + setLoadingText(false) + } + + // generateVoice + async function generateTestVoice() { + setLoadingVoice(true) + // TODO generateVoice() + setLoadingVoice(false) + } + + // generateImage + async function generateTestImage() { + setLoadingImage(true) + let description = 'test generate image' + const arrayBuffer = generateImage({ + modelName: null, + s: 'test', + }) + let imgArrayBuffer = await arrayBuffer(description) + console.log(imgArrayBuffer) // TODO create image + const blob = new Blob([imgArrayBuffer], { + type: 'image/png', + }) + const image = URL.createObjectURL(blob) + setGeneratedImage(image) + setLoadingImage(false) + } + + // generateDiffSound + async function generateTestDiffSound() { + setLoadingSound(true) + // TODO generateDiffSound() + setLoadingSound(false) + } + + // generateMotionDiffusion + async function generateTestMotionDiffusion() { + setLoadingMotion(true) + // TODO generateMotionDiffusion() + setLoadingMotion(false) + } + + // generateObjectOrConsumable + async function generateTestObjectOrConsumable() { + setLoadingObject(true) + // TODO generateObjectOrConsumable() + setLoadingObject(false) + } + + // generateGet3DObject + async function generateTestGet3DObject() { + setLoadingGet3DObject(true) + // TODO generateGet3DObject() + setLoadingGet3DObject(false) + } + + // generateMusic + async function generateTestMusic() { + setLoadingMusic(true) + // TODO generateMusic() + setLoadingMusic(false) + } + + // generateWeaviateCharacter + async function generateTestWeaviateCharacter() { + setLoadingCharacter(true) + // TODO generateWeaviateCharacter() + setLoadingCharacter(false) + } + + // generateSprite + async function generateTestSprite() { + setLoadingSprite(true) + // TODO generateSprite() + setLoadingSprite(false) + } + + // TODO styling!! + return ( +
+ + Test - Generators + + + +
+ + {loadingText &&

Loading...

} + {!loadingText && !generatedText &&

No data

} + {/* render result here */} +
+ + {loadingVoice &&

Loading...

} + {!loadingVoice && !generatedVoice &&

No data

} + {/* render result here */} +
+ + {loadingImage &&

Loading...

} + {!loadingImage && !generatedImage &&

No data

} + {!loadingImage && generatedImage && +
+
+ image +
+
+ } + + {loadingSound &&

Loading...

} + {!loadingSound && !generatedSound &&

No data

} + {/* render result here */} +
+ + {loadingMotion &&

Loading...

} + {!loadingMotion && !generatedMotion &&

No data

} + {/* render result here */} +
+ + {loadingObject &&

Loading...

} + {!loadingObject && !generatedObject &&

No data

} + {/* render result here */} +
+ + {loadingGet3DObject &&

Loading...

} + {!loadingGet3DObject && !generatedGet3DObject &&

No data

} + {/* render result here */} +
+ + {loadingMusic &&

Loading...

} + {!loadingMusic && !generatedMusic &&

No data

} + {/* render result here */} +
+ + {loadingCharacter &&

Loading...

} + {!loadingCharacter && !generatedCharacter &&

No data

} + {/* render result here */} +
+ + {loadingSprite &&

Loading...

} + {!loadingSprite && !generatedSprite &&

No data

} + {/* render result here */} +
+
+
+ ) +} From 7fe60dc028641b512904a0021cb0a63ba001abef Mon Sep 17 00:00:00 2001 From: listofbanned Date: Thu, 20 Oct 2022 18:59:52 -0300 Subject: [PATCH 2/8] Add voice --- constants/endpoints-constants.js | 1 + generators/generator.js | 27 ++++++------- pages/generators.js | 67 ++++++++++++++++++++++++++------ 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/constants/endpoints-constants.js b/constants/endpoints-constants.js index 689640f..097277c 100644 --- a/constants/endpoints-constants.js +++ b/constants/endpoints-constants.js @@ -1,5 +1,6 @@ export const gpt3Url = `https://gpt3.webaverse.com`; // check | create export const voiceUrl = `https://voice-cw.webaverse.com`; // test +// export const voiceUrl = `http://207.53.234.253` // coreweave export const stableDiffusionUrl = `https://stable-diffusion.webaverse.com`; export const diffsoundUrl = `https://diffsound.webaverse.com`; // test export const motionDiffusionUrl = `https://motion-diffusion.webaverse.com`; // check | create diff --git a/generators/generator.js b/generators/generator.js index 2c3ed24..4723a36 100644 --- a/generators/generator.js +++ b/generators/generator.js @@ -28,19 +28,20 @@ export const generateText = ({model}) => async ({x} = {}) => { }) } -export const generateVoice = ({model}) => async ({x} = {}) => { - const url = `${voiceUrl}/voice?x=${x}` // mock endpoint - await fetch(url) - .then(res => { - if (res.ok) { - // TODO: return generated voice - } else { - throw new Error(`invalid status: ${res.status}`); - } - }) - .catch(err => { - throw new Error(`url error: ${err}`); - }) +export const generateVoice = () => async ({s, voice} = {}) => { + return `${voiceUrl}/tts?s=${s}&voice=${voice}` + // const res = await fetch(url) + // .then(res => { + // if (res.ok) { + // return res + // } else { + // throw new Error(`invalid status: ${res.status}`); + // } + // return url + // }) + // .catch(err => { + // throw new Error(`url error: ${err}`); + // }) } export const generateImage = ({ diff --git a/pages/generators.js b/pages/generators.js index 5faddf6..214da77 100644 --- a/pages/generators.js +++ b/pages/generators.js @@ -14,12 +14,16 @@ import { generateSprite, } from '../generators/generator' +// import Reader from 'riff-wave-reader/lib/reader' + export default function Generators() { const [loadingText, setLoadingText] = useState(false) const [generatedText, setGeneratedText] = useState(null) const [loadingVoice, setLoadingVoice] = useState(false) const [generatedVoice, setGeneratedVoice] = useState(null) + const [transcript, setTranscript] = useState('') + const [voice, setVoice] = useState('') const [loadingImage, setLoadingImage] = useState(false) const [generatedImage, setGeneratedImage] = useState(null) @@ -53,9 +57,18 @@ export default function Generators() { } // generateVoice + const handleTranscript = e => { + setTranscript(e.target.value) + } + const handleVoice = e => { + setVoice(e.target.value) + } async function generateTestVoice() { - setLoadingVoice(true) - // TODO generateVoice() + const newVoice = generateVoice() + const voiceArrayBuffer = await newVoice({s: transcript, voice}) + const blob = new Blob([await (await fetch(voiceArrayBuffer)).arrayBuffer()]) + const audioFromBlob = URL.createObjectURL(blob) + setGeneratedVoice(audioFromBlob) setLoadingVoice(false) } @@ -68,7 +81,7 @@ export default function Generators() { s: 'test', }) let imgArrayBuffer = await arrayBuffer(description) - console.log(imgArrayBuffer) // TODO create image + const blob = new Blob([imgArrayBuffer], { type: 'image/png', }) @@ -147,13 +160,42 @@ export default function Generators() {
- {loadingVoice &&

Loading...

} + {loadingVoice && +
+ +
+ +
+ +
+ } {!loadingVoice && !generatedVoice &&

No data

} - {/* render result here */} + {!loadingVoice && generatedVoice && +
+
+ +
+
+ }
- {loadingSound &&

Loading...

} + {loadingSound && +
+ +
+ +
+ } {!loadingSound && !generatedSound &&

No data

} - {/* render result here */} + {!loadingSound && generatedSound && +
+
+ +
+
+ }
- {loadingText &&

Loading...

} - {!loadingText && !generatedText &&

No data

} - {/* render result here */} -
@@ -248,64 +195,47 @@ export default function Generators() { }
- {loadingMotion &&

Loading...

} - {!loadingMotion && !generatedMotion &&

No data

} - {/* render result here */} -
- - {loadingObject &&

Loading...

} - {!loadingObject && !generatedObject &&

No data

} - {/* render result here */} -
- - {loadingGet3DObject &&

Loading...

} - {!loadingGet3DObject && !generatedGet3DObject &&

No data

} - {/* render result here */} -
- - {loadingMusic &&

Loading...

} - {!loadingMusic && !generatedMusic &&

No data

} - {/* render result here */} -
- - {loadingCharacter &&

Loading...

} - {!loadingCharacter && !generatedCharacter &&

No data

} - {/* render result here */} + {loadingPixelArt &&

Loading, can take up to one minute...

} + {!loadingPixelArt && !generatedPixelArt &&

No data

} + {!loadingPixelArt && generatedPixelArt && +
+
+ image +
+
+ }
- {loadingSprite &&

Loading...

} - {!loadingSprite && !generatedSprite &&

No data

} - {/* render result here */} + {loadingBlip && +
+ +
+ +
+ } + {!loadingBlip && !generatedBlip &&

No data

} + {!loadingBlip && generatedBlip && +
+
+ {generatedBlip} +
+
+ }
From 34de20783705d4c5c050be346ea6ed9726411712 Mon Sep 17 00:00:00 2001 From: listofbanned Date: Tue, 1 Nov 2022 21:42:03 -0300 Subject: [PATCH 8/8] Add endpoints usage --- pages/generators.js | 59 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/pages/generators.js b/pages/generators.js index e147a02..51c1b0c 100644 --- a/pages/generators.js +++ b/pages/generators.js @@ -113,6 +113,14 @@ export default function Generators() {
+

Generate voice using tiktalknet

+
Endpoints: +

/tts?s={"{s}"}&voice={"{voice}"}

+
    +
  • s (string): text to convert
  • +
  • voice (string | optional): the id of the voice to use
  • +
+
} - {!loadingVoice && !generatedVoice &&

No data

} + {!loadingVoice && !generatedVoice &&

No voice data

} {!loadingVoice && generatedVoice &&

@@ -152,6 +160,14 @@ export default function Generators() {
}
+

Generate image using Stable Diffusion

+
Endpoints: +

/image?s={"{s}"}&model={"{model}"}

+
    +
  • s (string): image url
  • +
  • model (string | optional): the id of the model to use
  • +
+
@@ -185,7 +208,7 @@ export default function Generators() { } - {!loadingSound && !generatedSound &&

No data

} + {!loadingSound && !generatedSound &&

No sound data

} {!loadingSound && generatedSound &&

@@ -194,13 +217,19 @@ export default function Generators() {
}
+

Generate Pixel Art (AWS)

+
Endpoints: +

/generate : kick off a new image generation job and add it to the backlog. returns the id of the job

+

/generate_result : retrieve

+

/prompt_tags : returns the current tags added to prompts

+
{loadingPixelArt &&

Loading, can take up to one minute...

} - {!loadingPixelArt && !generatedPixelArt &&

No data

} + {!loadingPixelArt && !generatedPixelArt &&

No Pixel Art data

} {!loadingPixelArt && generatedPixelArt &&

@@ -209,6 +238,26 @@ export default function Generators() {
}
+

Generate image captioning using BLIP

+
Endpoints: +

POST /upload

+
    +
  • FormData task (string | optional): the task to run (image_captioning, vqa, feature_extraction or text_matching)
  • +
  • FormData file (file): the image to get the text caption
  • +
+

POST /upload/url

+

Body:

+

+ {"{"} + "task": {""}, + "file": {""} + {"}"} +

+
    +
  • Body task (string | optional): the task to run (image_captioning, vqa, feature_extraction or text_matching)
  • +
  • Body file (string): the image url to get the text caption
  • +
+
} - {!loadingBlip && !generatedBlip &&

No data

} + {!loadingBlip && !generatedBlip &&

No BLIP data

} {!loadingBlip && generatedBlip &&