prehistoric-systems/comixify
1
0
Fork 1
mirror of https://github.com/maciej3031/comixify.git synced 2026-03-11 08:54:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add pipeline selection

Browse source 
Change model files

Add frontend

Increase quality of I-frame selector

Add YouTube clips

Move defaults to settings

Fix Sports video
This commit is contained in:
adamsvystun 2018-10-07 10:26:31 +02:00 committed by Adam Svystun
parent d5ae415ce1
commit a68ec9fe38
13 changed files with 267 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
api/models.py
View file

@ -33,8 +33,8 @@ class Video(models.Model):
else:
raise TooLargeFile()
def create_comic(self):
keyframes = KeyFramesExtractor.get_keyframes(video=self)
def create_comic(self, frames_mode=0, rl_mode=0):
keyframes = KeyFramesExtractor.get_keyframes(video=self, frames_mode=frames_mode, rl_mode=rl_mode)
stylized_keyframes = StyleTransfer.get_stylized_frames(frames=keyframes)
comic_image = LayoutGenerator.get_layout(frames=stylized_keyframes)
return comic_image

11
api/serializers.py
View file

@ -2,13 +2,12 @@ from django.conf import settings
from rest_framework import serializers
from .exceptions import FileExtensionError, TooLargeFile
from .models import Video
class VideoSerializer(serializers.ModelSerializer):
class Meta:
model = Video
fields = ("file", "timestamp")
class VideoSerializer(serializers.Serializer):
file = serializers.FileField()
frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)
def validate(self, attrs):
file = attrs.get("file")
@ -21,3 +20,5 @@ class VideoSerializer(serializers.ModelSerializer):
class YouTubeDownloadSerializer(serializers.Serializer):
url = serializers.URLField()
frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)

10
api/views.py
View file

@ -19,7 +19,10 @@ class Comixify(APIView):
video_file = serializer.validated_data["file"]
video = Video.objects.create(file=video_file)
comic_image = video.create_comic()
comic_image = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"]
)
comic = Comic.create_from_nparray(comic_image, video)
response = {
@ -45,7 +48,10 @@ class ComixifyFromYoutube(APIView):
video = Video()
video.download_from_youtube(yt_url)
video.save()
comic_image = video.create_comic()
comic_image = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"]
)
comic = Comic.create_from_nparray(comic_image, video)
response = {

169
frontend/client/App.js
View file

@ -1,5 +1,6 @@
import React from "react";
import ReactDOM from "react-dom";
import YouTube from 'react-youtube';
import { post } from "axios";
import Dropzone from "react-dropzone";
import { BarLoader } from "react-spinners";
@ -18,27 +19,44 @@ class App extends React.Component {
PROCESSING: 1,
FINISHED: 2,
UPLOAD_ERROR: 3,
DROP_ERROR: 4
DROP_ERROR: 4,
SAMPLE_PROCESSING: 5
};
ytInput = React.createRef();
constructor(props) {
super(props);
this.state = {
state: App.appStates.INITIAL,
videoId: null,
drop_errors: [],
result_comics: null
result_comics: null,
framesMode: "0",
rlMode: "0"
};
this.onVideoDrop = this.onVideoDrop.bind(this);
this.onModelChange = this.onModelChange.bind(this);
this.handleResponse = this.handleResponse.bind(this);
this.onYouTubeSubmit = this.onYouTubeSubmit.bind(this);
this.onVideoUploadProgress = this.onVideoUploadProgress.bind(this);
this.onSamplingChange = this.onSamplingChange.bind(this);
}
onVideoUploadProgress(progressEvent) {
static onVideoUploadProgress(progressEvent) {
let percentCompleted = Math.round(
progressEvent.loaded * 100 / progressEvent.total
);
console.log(percentCompleted);
}
onModelChange(e) {
let value = e.currentTarget.value;
this.setState({
rlMode: value
})
}
onSamplingChange(e) {
let value = e.currentTarget.value;
this.setState({
framesMode: value
})
}
handleResponse(res) {
if (res.data["status_message"] === "ok") {
this.setState({
@ -52,11 +70,14 @@ class App extends React.Component {
}
}
processVideo(video) {
let { framesMode, rlMode } = this.state
let data = new FormData();
data.append("file", video);
data.set('frames_mode', parseInt(framesMode));
data.set('rl_mode', parseInt(rlMode));
post(COMIXIFY_API, data, {
headers: { "content-type": "multipart/form-data" },
onUploadProgress: this.onVideoUploadProgress
onUploadProgress: App.onVideoUploadProgress
})
.then(this.handleResponse)
.catch(err => {
@ -80,10 +101,12 @@ class App extends React.Component {
}
this.processVideo(files[0]);
}
onYouTubeSubmit() {
let ytLink = this.ytInput.current.value;
post(FROM_YOUTUBE_API, {
url: ytLink
submitYouTube(link) {
let { framesMode, rlMode } = this.state;
post(FROM_YOUTUBE_API, {
url: link,
frames_mode: parseInt(framesMode),
rl_mode: parseInt(rlMode)
})
.then(this.handleResponse)
.catch(err => {
@ -92,18 +115,34 @@ class App extends React.Component {
state: App.appStates.UPLOAD_ERROR
});
});
}
onYouTubeSubmit() {
let ytLink = this.ytInput.current.value;
this.submitYouTube(ytLink);
this.setState({
state: App.appStates.PROCESSING
});
}
onSamplePlay(videoId) {
let link = "https://www.youtube.com/watch?v=" + videoId;
this.submitYouTube(link);
this.setState({
videoId: videoId,
state: App.appStates.SAMPLE_PROCESSING
});
}
render() {
let { state, drop_errors, result_comics } = this.state;
let { state, drop_errors, result_comics, framesMode, rlMode, videoId } = this.state;
let showUsage = [
App.appStates.INITIAL,
App.appStates.UPLOAD_ERROR,
App.appStates.DROP_ERROR,
App.appStates.FINISHED
].includes(state);
let isProcessing = [
App.appStates.SAMPLE_PROCESSING,
App.appStates.PROCESSING
].includes(state);
return (
<div>
{state === App.appStates.FINISHED && [
@ -115,6 +154,53 @@ class App extends React.Component {
{state === App.appStates.UPLOAD_ERROR && (
<p>Server Error: Please try again later.</p>
)}
{showUsage && (
<div>
<div>Pipeline settings:</div>
<div>
<span>Frame sampling:</span>
<input
type="radio"
name="sampling"
id="sampling-0"
value="0"
checked={framesMode === "0"}
onChange={this.onSamplingChange}
/>
<label htmlFor="sampling-0">2fps sampling</label>
<input
type="radio"
name="sampling"
id="sampling-1"
value="1"
checked={framesMode === "1"}
onChange={this.onSamplingChange}
/>
<label htmlFor="sampling-1">I-frame sampling</label>
</div>
<div>
<span>Extraction model:</span>
<input
type="radio"
name="model"
id="model-0"
value="0"
checked={rlMode === "0"}
onChange={this.onModelChange}
/>
<label htmlFor="model-0">Basic model</label>
<input
type="radio"
name="model"
id="model-1"
value="1"
checked={rlMode === "1"}
onChange={this.onModelChange}
/>
<label htmlFor="model-1">+VTW model</label>
</div>
</div>
)}
{showUsage && (
<Dropzone
onDrop={this.onVideoDrop}
@ -133,13 +219,72 @@ class App extends React.Component {
<label htmlFor="yt-link" className="yt-label">Or use YouTube link:</label>
<input type="url" id="yt-link" ref={this.ytInput}/>
<button onClick={this.onYouTubeSubmit}>Run</button>
<div className="yt-clips-label">Or select one of sample videos:</div>
<div className="youtube-clips">
<div>
<div className="yt-clip-label">Documentary</div>
<YouTube
videoId="gr1ps0ooDhU"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "gr1ps0ooDhU")}
/>
</div>
<div>
<div className="yt-clip-label">Sports</div>
<YouTube
videoId="MqqyD0nP1LQ"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "MqqyD0nP1LQ")}
/>
</div>
<div>
<div className="yt-clip-label">Music video</div>
<YouTube
videoId="kJQP7kiw5Fk"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "kJQP7kiw5Fk")}
/>
</div>
<div>
<div className="yt-clip-label">Politics</div>
<YouTube
videoId="F2b-2YnfZso"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "F2b-2YnfZso")}
/>
</div>
</div>
</div>
)}
{state === App.appStates.PROCESSING && (
{state === App.appStates.SAMPLE_PROCESSING && (
<YouTube
videoId={videoId}
opts={{
height: '390',
width: '640',
playerVars: {
autoplay: 1
}
}}
/>
)}
{isProcessing && (
<BarLoader
color={"rgb(54, 215, 183)"}
className={css`
margin: 0 auto;
margin: 20px auto 0 auto;
`}
width={10}
widthUnit="rem"

9
frontend/client/webpack.config.js
View file

@ -1,7 +1,8 @@
const merge = require("webpack-merge");
const UglifyJsPlugin = require("uglifyjs-webpack-plugin");
const webpack = require("webpack");
module.exports = env => {
const merge = require("webpack-merge");
const UglifyJsPlugin = require("uglifyjs-webpack-plugin");
const webpack = require("webpack");
let common = {
entry: {
@ -35,7 +36,7 @@ module.exports = env => {
}
};
if (env.production) {
if (env && env.production) {
return merge(common, {
plugins: [
new webpack.DefinePlugin({

16
frontend/static/frontend/css/app.css
View file

@ -61,7 +61,7 @@ img {
background-color: #eee;
}
.yt-label {
.yt-label, .yt-clips-label {
margin: 10px 0 10px 20px;
display: block;
}
@ -111,6 +111,20 @@ button:focus {
outline: 0;
}
.youtube-clips {
display: flex;
justify-content: space-between;
flex-wrap: wrap;
}
.youtube-clips > div {
}
.yt-clip-label {
}
@media all and (max-width: 700px) {
.wrap {
margin-top: 0;

14
frontend/static/frontend/js/app.client.js
View file

File diff suppressed because one or more lines are too long

32
keyframes/keyframes.py
View file

@ -25,25 +25,34 @@ logger = logging.getLogger(__name__)
class KeyFramesExtractor:
@classmethod
def get_keyframes(cls, video, gpu=settings.GPU, features_batch_size=settings.FEATURE_BATCH_SIZE):
frames_paths, all_frames_tmp_dir = cls._get_all_frames(video)
def get_keyframes(cls, video, gpu=settings.GPU, features_batch_size=settings.FEATURE_BATCH_SIZE,
frames_mode=0, rl_mode=0):
frames_paths, all_frames_tmp_dir = cls._get_all_frames(video, mode=frames_mode)
frames = cls._get_frames(frames_paths)
features = cls._get_features(frames, gpu, features_batch_size)
change_points, frames_per_segment = cls._get_segments(features)
probs = cls._get_probs(features, gpu)
probs = cls._get_probs(features, gpu, mode=rl_mode)
chosen_frames = cls._get_chosen_frames(frames, probs, change_points, frames_per_segment)
return chosen_frames
@staticmethod
def _get_all_frames(video):
def _get_all_frames(video, mode=0):
all_frames_tmp_dir = uuid.uuid4().hex
os.mkdir(jj(settings.TMP_DIR, all_frames_tmp_dir))
call(["ffmpeg", "-i", video.file.path, "-vf", "select=not(mod(n\\,15))", "-vsync", "vfr", "-q:v", "2",
jj(settings.TMP_DIR, all_frames_tmp_dir, "%06d.jpeg")])
if mode == 1:
call(["ffmpeg", "-i", f"{video.file.path}", "-c:v", "libxvid", "-qscale:v", "1", "-an",
jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "video.mp4")])
call(["ffmpeg", "-i", jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "video.mp4"), "-vf",
"select=eq(pict_type\,I)", "-vsync", "vfr",
jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "%06d.jpeg")])
else:
call(["ffmpeg", "-i", video.file.path, "-vf", "select=not(mod(n\\,15))", "-vsync", "vfr", "-q:v", "2",
jj(settings.TMP_DIR, all_frames_tmp_dir, "%06d.jpeg")])
frames_paths = []
for dirname, dirnames, filenames in os.walk(jj(settings.TMP_DIR, all_frames_tmp_dir)):
for filename in filenames:
frames_paths.append(jj(dirname, filename))
if not filename.endswith(".mp4"):
frames_paths.append(jj(dirname, filename))
return sorted(frames_paths), all_frames_tmp_dir
@staticmethod
@ -92,12 +101,15 @@ class KeyFramesExtractor:
return features.astype(np.float32)
@staticmethod
def _get_probs(features, gpu=True):
model_cache_key = "keyframes_rl_model_cache"
def _get_probs(features, gpu=True, mode=0):
model_cache_key = "keyframes_rl_model_cache_" + str(mode)
model = cache.get(model_cache_key) # get model from cache
if model is None:
model_path = "keyframes_rl/pretrained_model/model_epoch100.pth.tar"
if mode == 1:
model_path = "keyframes_rl/pretrained_model/model_1.pth.tar"
else:
model_path = "keyframes_rl/pretrained_model/model_0.pth.tar"
model = DSN(in_dim=1024, hid_dim=256, num_layers=1, cell="lstm")
if gpu:
checkpoint = torch.load(model_path)

BIN
keyframes_rl/pretrained_model/model_0.pth.tar Normal file
View file

Binary file not shown.

BIN
keyframes_rl/pretrained_model/model_epoch100.pth.tar → keyframes_rl/pretrained_model/model_1.pth.tar
View file

Binary file not shown.

40
package-lock.json generated
View file

@ -3953,6 +3953,11 @@
"invert-kv": "1.0.0"
}
},
"load-script": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/load-script/-/load-script-1.0.0.tgz",
"integrity": "sha1-BJGTngvuVkPuSUp+PaPSuscMbKQ="
},
"loader-runner": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.3.1.tgz",
@ -5271,6 +5276,16 @@
"recompose": "0.27.1"
}
},
"react-youtube": {
"version": "7.7.0",
"resolved": "https://registry.npmjs.org/react-youtube/-/react-youtube-7.7.0.tgz",
"integrity": "sha512-ZHHG3x7y9P8oCldPx0t4z0jTK9GC4lBVzKnYcd8SHQe2x5mCUIxLNWXzR8+Oe7H2I4ACCB87lLF5WuU39PGuCw==",
"requires": {
"lodash": "4.17.10",
"prop-types": "15.6.2",
"youtube-player": "5.5.1"
}
},
"readable-stream": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
@ -5629,6 +5644,11 @@
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
"dev": true
},
"sister": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/sister/-/sister-3.0.1.tgz",
"integrity": "sha512-aG41gNRHRRxPq52MpX4vtm9tapnr6ENmHUx8LMAJWCOplEMwXzh/dp5WIo52Wl8Zlc/VUyHLJ2snX0ck+Nma9g=="
},
"slash": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
@ -6637,6 +6657,26 @@
"requires": {
"camelcase": "4.1.0"
}
},
"youtube-player": {
"version": "5.5.1",
"resolved": "https://registry.npmjs.org/youtube-player/-/youtube-player-5.5.1.tgz",
"integrity": "sha512-1d62W9She0B1uKNyY6K7jtWFbOW3dYsm6hyKzrh11BLOuYFzkt8K6AcQ3QdPF3aU47dzhZ82clzOJVVWus4HTw==",
"requires": {
"debug": "2.6.9",
"load-script": "1.0.0",
"sister": "3.0.1"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
}
}
}
}
}

1
package.json
View file

@ -34,6 +34,7 @@
"react-dom": "^16.4.2",
"react-dropzone": "^5.0.1",
"react-spinners": "^0.4.5",
"react-youtube": "^7.7.0",
"style-loader": "^0.23.0",
"uglifyjs-webpack-plugin": "^1.2.5"
},

2
settings/settings.py
View file

@ -142,3 +142,5 @@ TMP_DIR = 'tmp/'
GPU = True
FEATURE_BATCH_SIZE = 32
DEFAULT_FRAMES_SAMPLING_MODE = 0
DEFAULT_RL_MODE = 0