prehistoric-systems/comixify
1
0
Fork 1
mirror of https://github.com/maciej3031/comixify.git synced 2026-03-11 08:54:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add get comix endpoint (#18)

Browse source 
* Add endpoint to getting images

* Add migration

* Fix minor bug

* Fix another bug

* Fix problem with CartoonGAN not change model and add timestamp to Comic

* Add minimum response delay
Update sample
This commit is contained in:
Maciej Pęśko 2018-11-18 13:48:39 +01:00 committed by GitHub
parent 945625d1d7
commit ffc6b2dc28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 232 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
api/migrations/0002_add_additional_info_to_Comic.py Normal file
View file

@ -0,0 +1,38 @@
# Generated by Django 2.0.7 on 2018-11-18 00:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('api', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='comic',
name='frames_mode',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='comic',
name='image_assessment_mode',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='comic',
name='rl_mode',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='comic',
name='style_transfer_mode',
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name='comic',
name='yt_url',
field=models.URLField(blank=True, null=True),
),
]

18
api/migrations/0003_add_timestamp.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 2.0.7 on 2018-11-18 10:55
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('api', '0002_add_additional_info_to_Comic'),
]
operations = [
migrations.AddField(
model_name='comic',
name='timestamp',
field=models.DateTimeField(auto_now_add=True),
),
]

34
api/models.py
View file

@ -34,7 +34,7 @@ class Video(models.Model):
else:
raise TooLargeFile()
def create_comic(self, frames_mode=0, rl_mode=0, image_assessment_mode=0, style_transfer_mode=0):
def create_comix(self, yt_url='', frames_mode=0, rl_mode=0, image_assessment_mode=0, style_transfer_mode=0):
(keyframes, keyframes_timings), keyframes_extraction_time = KeyFramesExtractor.get_keyframes(
video=self,
frames_mode=frames_mode,
@ -45,29 +45,49 @@ class Video(models.Model):
style_transfer_mode=style_transfer_mode)
comic_image, layout_generation_time = LayoutGenerator.get_layout(frames=stylized_keyframes)
comic, from_nparray_time = Comic.create_from_nparray(nparray=comic_image,
video=self,
yt_url=yt_url,
frames_mode=frames_mode,
rl_mode=rl_mode,
image_assessment_mode=image_assessment_mode,
style_transfer_mode=style_transfer_mode)
timings = {
'from_nparray_time': from_nparray_time,
'keyframes_extraction_time': keyframes_extraction_time,
'stylization_time': stylization_time,
'layout_generation_time': layout_generation_time,
'keyframes_extraction_time_details': keyframes_timings
}
return comic_image, timings
return comic, timings
class Comic(models.Model):
file = models.FileField(blank=False, null=False, upload_to="comic")
video = models.ForeignKey(Video, on_delete=models.CASCADE, related_name="comic")
yt_url = models.URLField(blank=True, null=True)
frames_mode = models.PositiveIntegerField(default=0)
rl_mode = models.PositiveIntegerField(default=0)
image_assessment_mode = models.PositiveIntegerField(default=0)
style_transfer_mode = models.PositiveIntegerField(default=0)
timestamp = models.DateTimeField(auto_now_add=True)
@classmethod
@profile
def create_from_nparray(cls, nparray_file, video):
if nparray_file.max() <= 1:
nparray_file = (nparray_file).astype(int)
def create_from_nparray(cls, nparray, video, yt_url, frames_mode,
rl_mode, image_assessment_mode, style_transfer_mode):
tmp_name = uuid.uuid4().hex + ".png"
cv2.imwrite(jj(settings.TMP_DIR, tmp_name), nparray_file)
cv2.imwrite(jj(settings.TMP_DIR, tmp_name), nparray)
with open(jj(settings.TMP_DIR, tmp_name), mode="rb") as tmp_file:
comic_image = File(tmp_file, name=tmp_name)
comic = Comic.objects.create(file=comic_image, video=video)
comic = Comic.objects.create(file=comic_image,
video=video,
yt_url=yt_url,
frames_mode=frames_mode,
rl_mode=rl_mode,
image_assessment_mode=image_assessment_mode,
style_transfer_mode=style_transfer_mode)
os.remove(jj(settings.TMP_DIR, tmp_name))
return comic

4
api/serializers.py
View file

@ -16,9 +16,9 @@ class VideoSerializer(serializers.Serializer):
def validate(self, attrs):
file = attrs.get("file")
if file.name.split(".")[-1] not in settings.PERMITTED_VIDEO_EXTENSIONS:
raise FileExtensionError
raise FileExtensionError()
if file.size > settings.MAX_FILE_SIZE:
raise TooLargeFile
raise TooLargeFile()
return attrs

75
api/views.py
View file

@ -16,21 +16,24 @@ class Comixify(APIView):
serializer = VideoSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
video_file = serializer.validated_data["file"]
frames_mode = serializer.validated_data["frames_mode"]
rl_mode = serializer.validated_data["rl_mode"]
image_assessment_mode = serializer.validated_data["image_assessment_mode"]
style_transfer_mode = serializer.validated_data["style_transfer_mode"]
video = Video.objects.create(file=video_file)
comic_image, timings = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"],
image_assessment_mode=serializer.validated_data["image_assessment_mode"],
style_transfer_mode=serializer.validated_data["style_transfer_mode"],
comix, timings = video.create_comix(
yt_url='',
frames_mode=frames_mode,
rl_mode=rl_mode,
image_assessment_mode=image_assessment_mode,
style_transfer_mode=style_transfer_mode
)
comic, from_nparray_time = Comic.create_from_nparray(comic_image, video)
timings['from_nparray_time'] = from_nparray_time
response = {
"status_message": "ok",
"comic": comic.file.url,
"comic": comix.file.url,
"timings": timings,
}
# Remove to spare storage
@ -39,7 +42,6 @@ class Comixify(APIView):
class ComixifyFromYoutube(APIView):
def post(self, request):
"""
Receives video, and returns comic image
@ -48,25 +50,40 @@ class ComixifyFromYoutube(APIView):
serializer = YouTubeDownloadSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
yt_url = serializer.validated_data["url"]
frames_mode = serializer.validated_data["frames_mode"]
rl_mode = serializer.validated_data["rl_mode"]
image_assessment_mode = serializer.validated_data["image_assessment_mode"]
style_transfer_mode = serializer.validated_data["style_transfer_mode"]
video = Video()
_, yt_download_time = video.download_from_youtube(yt_url)
video.save()
comic_image, timings = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"],
image_assessment_mode=serializer.validated_data["image_assessment_mode"],
style_transfer_mode=serializer.validated_data["style_transfer_mode"],
)
comic, from_nparray_time = Comic.create_from_nparray(comic_image, video)
timings['from_nparray_time'] = from_nparray_time
timings['yt_download_time'] = yt_download_time
try:
comix = Comic.objects.filter(yt_url=yt_url,
frames_mode=frames_mode,
rl_mode=rl_mode,
image_assessment_mode=image_assessment_mode,
style_transfer_mode=style_transfer_mode
).latest('timestamp')
response = {
"status_message": "ok",
"comic": comix.file.url,
}
except Comic.DoesNotExist:
video = Video()
_, yt_download_time = video.download_from_youtube(yt_url)
video.save()
comix, timings = video.create_comix(
yt_url=yt_url,
frames_mode=frames_mode,
rl_mode=rl_mode,
image_assessment_mode=image_assessment_mode,
style_transfer_mode=style_transfer_mode
)
response = {
"status_message": "ok",
"comic": comic.file.url,
"timings": timings,
}
# Remove to spare storage
video.file.delete()
timings['yt_download_time'] = yt_download_time
response = {
"status_message": "ok",
"comic": comix.file.url,
"timings": timings,
}
# Remove to spare storage
video.file.delete()
return Response(response)

17
frontend/client/App.js
View file

@ -5,12 +5,14 @@ import { post } from "axios";
import Dropzone from "react-dropzone";
import { BarLoader } from "react-spinners";
import { css } from "react-emotion";
import debounce from "lodash/debounce"
import {
COMIXIFY_API,
MAX_FILE_SIZE,
PERMITTED_VIDEO_EXTENSIONS,
FROM_YOUTUBE_API
FROM_YOUTUBE_API,
MIN_RESPONSE_DELAY
} from "./constants";
class App extends React.Component {
@ -37,7 +39,7 @@ class App extends React.Component {
};
this.onVideoDrop = this.onVideoDrop.bind(this);
this.onModelChange = this.onModelChange.bind(this);
this.handleResponse = this.handleResponse.bind(this);
this.handleResponse = debounce(this.handleResponse.bind(this), MIN_RESPONSE_DELAY);
this.onYouTubeSubmit = this.onYouTubeSubmit.bind(this);
this.onSamplingChange = this.onSamplingChange.bind(this);
this.onImageAssessmentChange = this.onImageAssessmentChange.bind(this);
@ -74,6 +76,9 @@ class App extends React.Component {
})
}
handleResponse(res) {
if(res === "debounce") {
return
}
if (res.data["status_message"] === "ok") {
this.setState({
state: App.appStates.FINISHED,
@ -93,6 +98,7 @@ class App extends React.Component {
data.set('rl_mode', parseInt(rlMode));
data.set("image_assessment_mode", parseInt(imageAssessment));
data.set('style_transfer_mode', parseInt(styleTransferMode));
this.handleResponse("debounce");
post(COMIXIFY_API, data, {
headers: { "content-type": "multipart/form-data" },
onUploadProgress: App.onVideoUploadProgress
@ -121,6 +127,7 @@ class App extends React.Component {
}
submitYouTube(link) {
let { framesMode, rlMode, imageAssessment, styleTransferMode } = this.state;
this.handleResponse("debounce");
post(FROM_YOUTUBE_API, {
url: link,
frames_mode: parseInt(framesMode),
@ -317,14 +324,14 @@ class App extends React.Component {
/>
</div>
<div>
<div className="yt-clip-label">Sport</div>
<div className="yt-clip-label">Music</div>
<YouTube
videoId="Pi-qitmfvlI"
videoId="Es3Vsfzdr14"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "Pi-qitmfvlI")}
onPlay={this.onSamplePlay.bind(this, "Es3Vsfzdr14")}
/>
</div>
<div>

1
frontend/client/constants.js
View file

@ -2,3 +2,4 @@ export const COMIXIFY_API = "/comixify/";
export const FROM_YOUTUBE_API = "/comixify/from_yt/";
export const MAX_FILE_SIZE = 50000000;
export const PERMITTED_VIDEO_EXTENSIONS = "video/*";
export const MIN_RESPONSE_DELAY = 7500;

4
frontend/static/frontend/css/app.css
View file

@ -35,6 +35,10 @@ img {
width: 100%;
}
#demo {
overflow: hidden;
}
.bar-loader {
margin: 0 auto;
}

12
frontend/static/frontend/js/app.client.js
View file

File diff suppressed because one or more lines are too long

43
get_yt_comix_media_urls.py Normal file
View file

@ -0,0 +1,43 @@
import pandas as pd
import requests
yt_urls = {'pulp_fiction': 'https://www.youtube.com/watch?v=pvAhRcUofDk'}
base_comixify_url = 'https://comixify.ii.pw.edu.pl'
frames_modes = [0, 1]
rl_modes = [0, 1]
image_assessment_modes = [0, 1]
style_transfer_modes = [0, 1, 2]
results = {'clip_name': [],
'clip_yt_url': [],
'media_url': [],
'frames_mode': [],
'rl_mode': [],
'image_assessment_mode': [],
'style_transfer_mode': []}
for clip_name in yt_urls.keys():
for f_mode in frames_modes:
for rl_model in rl_modes:
for iam_mode in image_assessment_modes:
for st_mode in style_transfer_modes:
r = requests.post(base_comixify_url + '/comixify/from_yt/',
data={'url': yt_urls[clip_name],
'frames_mode': f_mode,
'rl_mode': rl_model,
'image_assessment_mode': iam_mode,
'style_transfer_mode': st_mode})
media_url = base_comixify_url + r.json()['comic']
results['clip_name'].append(clip_name)
results['clip_yt_url'].append(yt_urls[clip_name])
results['media_url'].append(media_url)
results['frames_mode'].append(f_mode)
results['rl_mode'].append(rl_model)
results['image_assessment_mode'].append(iam_mode)
results['style_transfer_mode'].append(st_mode)
df = pd.DataFrame(results)
df.to_csv('yt_comix_media_urls')

34
package-lock.json generated
View file

@ -10,7 +10,7 @@
"integrity": "sha1-zgBCgEX7t9XrwOp7+DV4nxU2arI=",
"requires": {
"@babel/types": "7.0.0-beta.51",
"lodash": "4.17.10"
"lodash": "4.17.11"
}
},
"@babel/types": {
@ -19,7 +19,7 @@
"integrity": "sha1-2AK3tUO1g2x3iqaReXq/APPZfqk=",
"requires": {
"esutils": "2.0.2",
"lodash": "4.17.10",
"lodash": "4.17.11",
"to-fast-properties": "2.0.0"
},
"dependencies": {
@ -505,7 +505,7 @@
"convert-source-map": "1.5.1",
"debug": "2.6.9",
"json5": "0.5.1",
"lodash": "4.17.10",
"lodash": "4.17.11",
"minimatch": "3.0.4",
"path-is-absolute": "1.0.1",
"private": "0.1.8",
@ -533,7 +533,7 @@
"babel-types": "6.26.0",
"detect-indent": "4.0.0",
"jsesc": "1.3.0",
"lodash": "4.17.10",
"lodash": "4.17.11",
"source-map": "0.5.7",
"trim-right": "1.0.1"
}
@ -587,7 +587,7 @@
"babel-helper-function-name": "6.24.1",
"babel-runtime": "6.26.0",
"babel-types": "6.26.0",
"lodash": "4.17.10"
"lodash": "4.17.11"
}
},
"babel-helper-explode-assignable-expression": {
@ -657,7 +657,7 @@
"requires": {
"babel-runtime": "6.26.0",
"babel-types": "6.26.0",
"lodash": "4.17.10"
"lodash": "4.17.11"
}
},
"babel-helper-remap-async-to-generator": {
@ -905,7 +905,7 @@
"babel-template": "6.26.0",
"babel-traverse": "6.26.0",
"babel-types": "6.26.0",
"lodash": "4.17.10"
"lodash": "4.17.11"
}
},
"babel-plugin-transform-es2015-classes": {
@ -1295,7 +1295,7 @@
"babel-runtime": "6.26.0",
"core-js": "2.5.7",
"home-or-tmp": "2.0.0",
"lodash": "4.17.10",
"lodash": "4.17.11",
"mkdirp": "0.5.1",
"source-map-support": "0.4.18"
}
@ -1318,7 +1318,7 @@
"babel-traverse": "6.26.0",
"babel-types": "6.26.0",
"babylon": "6.18.0",
"lodash": "4.17.10"
"lodash": "4.17.11"
}
},
"babel-traverse": {
@ -1334,7 +1334,7 @@
"debug": "2.6.9",
"globals": "9.18.0",
"invariant": "2.2.4",
"lodash": "4.17.10"
"lodash": "4.17.11"
},
"dependencies": {
"debug": {
@ -1354,7 +1354,7 @@
"requires": {
"babel-runtime": "6.26.0",
"esutils": "2.0.2",
"lodash": "4.17.10",
"lodash": "4.17.11",
"to-fast-properties": "1.0.3"
}
},
@ -3610,7 +3610,7 @@
"cli-width": "2.2.0",
"external-editor": "3.0.3",
"figures": "2.0.0",
"lodash": "4.17.10",
"lodash": "4.17.11",
"mute-stream": "0.0.7",
"run-async": "2.3.0",
"rxjs": "6.3.1",
@ -3984,9 +3984,9 @@
}
},
"lodash": {
"version": "4.17.10",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz",
"integrity": "sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg=="
"version": "4.17.11",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
},
"lodash.camelcase": {
"version": "4.3.0",
@ -5281,7 +5281,7 @@
"resolved": "https://registry.npmjs.org/react-youtube/-/react-youtube-7.7.0.tgz",
"integrity": "sha512-ZHHG3x7y9P8oCldPx0t4z0jTK9GC4lBVzKnYcd8SHQe2x5mCUIxLNWXzR8+Oe7H2I4ACCB87lLF5WuU39PGuCw==",
"requires": {
"lodash": "4.17.10",
"lodash": "4.17.11",
"prop-types": "15.6.2",
"youtube-player": "5.5.1"
}
@ -6465,7 +6465,7 @@
"integrity": "sha512-TmSe1HZKeOPey3oy1Ov2iS3guIZjWvMT2BBJDzzT5jScHTjVC3mpjJofgueEzaEd6ibhxRDD6MIblDr8tzh8iQ==",
"dev": true,
"requires": {
"lodash": "4.17.10"
"lodash": "4.17.11"
}
},
"webpack-sources": {

1
package.json
View file

@ -30,6 +30,7 @@
"babel-preset-stage-0": "^6.3.13",
"babel-register": "^6.18.0",
"css-loader": "^0.28.7",
"lodash": "^4.17.11",
"react": "^16.4.2",
"react-dom": "^16.4.2",
"react-dropzone": "^5.0.1",

8
requirements.txt
View file

@ -1,4 +1,7 @@
absl-py==0.6.1
astor==0.7.1
certifi==2018.10.15
chardet==3.0.4
cloudpickle==0.6.1
cycler==0.10.0
dask==0.20.0
@ -10,6 +13,7 @@ gast==0.2.0
grpcio==1.16.0
gunicorn==19.9.0
h5py==2.8.0
idna==2.7
Keras==2.2.4
Keras-Applications==1.0.6
Keras-Preprocessing==1.0.5
@ -21,6 +25,7 @@ numpy==1.14.5
opencv-contrib-python==3.4.3.18
opencv-python==3.4.2.17
pafy==0.5.4
pandas==0.23.4
Pillow==5.2.0
protobuf==3.6.1
psycopg2==2.7.5
@ -29,6 +34,7 @@ python-dateutil==2.7.5
pytz==2018.5
PyWavelets==1.0.1
PyYAML==3.13
requests==2.20.1
scikit-image==0.14.1
scikit-learn==0.20.0
scipy==1.1.0
@ -40,6 +46,6 @@ termcolor==1.1.0
toolz==0.9.0
torch==0.4.1
torchvision==0.2.1
urllib3==1.24.1
Werkzeug==0.14.1
youtube-dl==2018.11.7
tensorflow-gpu==1.10.1

12
style_transfer/style_transfer.py
View file

@ -62,8 +62,16 @@ class StyleTransfer():
@classmethod
def _cartoon_gan_stylize(cls, frames, gpu=True, style='Hayao'):
model_cache_key = 'model_cache'
model = cache.get(model_cache_key) # get model from cache
if style == 'Hayao':
model_cache_key = 'model_cache_hayao'
model = cache.get(model_cache_key) # get model from cache
elif style == 'Hosoda':
model_cache_key = 'model_cache_hosoda'
model = cache.get(model_cache_key) # get model from cache
else:
raise Exception('No such CartoonGAN model!')
if model is None:
# load pretrained model