cOborski/comixify
1
0
Fork 0
forked from prehistoric-systems/comixify
Code Pull requests Activity

Merge pull request #8 from maciej3031/frame_pipeline_selection

Browse source 
Frame pipeline selection
This commit is contained in:
Adam Svystun 2018-10-08 21:45:54 +02:00 committed by GitHub
commit 18d895e7ff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 267 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
api/models.py
View file

@ -33,8 +33,8 @@ class Video(models.Model):
else:
raise TooLargeFile()
def create_comic(self):
keyframes = KeyFramesExtractor.get_keyframes(video=self)
def create_comic(self, frames_mode=0, rl_mode=0):
keyframes = KeyFramesExtractor.get_keyframes(video=self, frames_mode=frames_mode, rl_mode=rl_mode)
stylized_keyframes = StyleTransfer.get_stylized_frames(frames=keyframes)
comic_image = LayoutGenerator.get_layout(frames=stylized_keyframes)
return comic_image

11
api/serializers.py
View file

@ -2,13 +2,12 @@ from django.conf import settings
from rest_framework import serializers
from .exceptions import FileExtensionError, TooLargeFile
from .models import Video
class VideoSerializer(serializers.ModelSerializer):
class Meta:
model = Video
fields = ("file", "timestamp")
class VideoSerializer(serializers.Serializer):
file = serializers.FileField()
frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)
def validate(self, attrs):
file = attrs.get("file")
@ -21,3 +20,5 @@ class VideoSerializer(serializers.ModelSerializer):
class YouTubeDownloadSerializer(serializers.Serializer):
url = serializers.URLField()
frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)

10
api/views.py
View file

@ -19,7 +19,10 @@ class Comixify(APIView):
video_file = serializer.validated_data["file"]
video = Video.objects.create(file=video_file)
comic_image = video.create_comic()
comic_image = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"]
)
comic = Comic.create_from_nparray(comic_image, video)
response = {
@ -45,7 +48,10 @@ class ComixifyFromYoutube(APIView):
video = Video()
video.download_from_youtube(yt_url)
video.save()
comic_image = video.create_comic()
comic_image = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"]
)
comic = Comic.create_from_nparray(comic_image, video)
response = {

169
frontend/client/App.js
View file

@ -1,5 +1,6 @@
import React from "react";
import ReactDOM from "react-dom";
import YouTube from 'react-youtube';
import { post } from "axios";
import Dropzone from "react-dropzone";
import { BarLoader } from "react-spinners";
@ -18,27 +19,44 @@ class App extends React.Component {
PROCESSING: 1,
FINISHED: 2,
UPLOAD_ERROR: 3,
DROP_ERROR: 4
DROP_ERROR: 4,
SAMPLE_PROCESSING: 5
};
ytInput = React.createRef();
constructor(props) {
super(props);
this.state = {
state: App.appStates.INITIAL,
videoId: null,
drop_errors: [],
result_comics: null
result_comics: null,
framesMode: "0",
rlMode: "0"
};
this.onVideoDrop = this.onVideoDrop.bind(this);
this.onModelChange = this.onModelChange.bind(this);
this.handleResponse = this.handleResponse.bind(this);
this.onYouTubeSubmit = this.onYouTubeSubmit.bind(this);
this.onVideoUploadProgress = this.onVideoUploadProgress.bind(this);
this.onSamplingChange = this.onSamplingChange.bind(this);
}
onVideoUploadProgress(progressEvent) {
static onVideoUploadProgress(progressEvent) {
let percentCompleted = Math.round(
progressEvent.loaded * 100 / progressEvent.total
);
console.log(percentCompleted);
}
onModelChange(e) {
let value = e.currentTarget.value;
this.setState({
rlMode: value
})
}
onSamplingChange(e) {
let value = e.currentTarget.value;
this.setState({
framesMode: value
})
}
handleResponse(res) {
if (res.data["status_message"] === "ok") {
this.setState({
@ -52,11 +70,14 @@ class App extends React.Component {
}
}
processVideo(video) {
let { framesMode, rlMode } = this.state
let data = new FormData();
data.append("file", video);
data.set('frames_mode', parseInt(framesMode));
data.set('rl_mode', parseInt(rlMode));
post(COMIXIFY_API, data, {
headers: { "content-type": "multipart/form-data" },
onUploadProgress: this.onVideoUploadProgress
onUploadProgress: App.onVideoUploadProgress
})
.then(this.handleResponse)
.catch(err => {
@ -80,10 +101,12 @@ class App extends React.Component {
}
this.processVideo(files[0]);
}
onYouTubeSubmit() {
let ytLink = this.ytInput.current.value;
post(FROM_YOUTUBE_API, {
url: ytLink
submitYouTube(link) {
let { framesMode, rlMode } = this.state;
post(FROM_YOUTUBE_API, {
url: link,
frames_mode: parseInt(framesMode),
rl_mode: parseInt(rlMode)
})
.then(this.handleResponse)
.catch(err => {
@ -92,18 +115,34 @@ class App extends React.Component {
state: App.appStates.UPLOAD_ERROR
});
});
}
onYouTubeSubmit() {
let ytLink = this.ytInput.current.value;
this.submitYouTube(ytLink);
this.setState({
state: App.appStates.PROCESSING
});
}
onSamplePlay(videoId) {
let link = "https://www.youtube.com/watch?v=" + videoId;
this.submitYouTube(link);
this.setState({
videoId: videoId,
state: App.appStates.SAMPLE_PROCESSING
});
}
render() {
let { state, drop_errors, result_comics } = this.state;
let { state, drop_errors, result_comics, framesMode, rlMode, videoId } = this.state;
let showUsage = [
App.appStates.INITIAL,
App.appStates.UPLOAD_ERROR,
App.appStates.DROP_ERROR,
App.appStates.FINISHED
].includes(state);
let isProcessing = [
App.appStates.SAMPLE_PROCESSING,
App.appStates.PROCESSING
].includes(state);
return (
<div>
{state === App.appStates.FINISHED && [
@ -115,6 +154,53 @@ class App extends React.Component {
{state === App.appStates.UPLOAD_ERROR && (
<p>Server Error: Please try again later.</p>
)}
{showUsage && (
<div>
<div>Pipeline settings:</div>
<div>
<span>Frame sampling:</span>
<input
type="radio"
name="sampling"
id="sampling-0"
value="0"
checked={framesMode === "0"}
onChange={this.onSamplingChange}
/>
<label htmlFor="sampling-0">2fps sampling</label>
<input
type="radio"
name="sampling"
id="sampling-1"
value="1"
checked={framesMode === "1"}
onChange={this.onSamplingChange}
/>
<label htmlFor="sampling-1">I-frame sampling</label>
</div>
<div>
<span>Extraction model:</span>
<input
type="radio"
name="model"
id="model-0"
value="0"
checked={rlMode === "0"}
onChange={this.onModelChange}
/>
<label htmlFor="model-0">Basic model</label>
<input
type="radio"
name="model"
id="model-1"
value="1"
checked={rlMode === "1"}
onChange={this.onModelChange}
/>
<label htmlFor="model-1">+VTW model</label>
</div>
</div>
)}
{showUsage && (
<Dropzone
onDrop={this.onVideoDrop}
@ -133,13 +219,72 @@ class App extends React.Component {
<label htmlFor="yt-link" className="yt-label">Or use YouTube link:</label>
<input type="url" id="yt-link" ref={this.ytInput}/>
<button onClick={this.onYouTubeSubmit}>Run</button>
<div className="yt-clips-label">Or select one of sample videos:</div>
<div className="youtube-clips">
<div>
<div className="yt-clip-label">Documentary</div>
<YouTube
videoId="gr1ps0ooDhU"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "gr1ps0ooDhU")}
/>
</div>
<div>
<div className="yt-clip-label">Sports</div>
<YouTube
videoId="MqqyD0nP1LQ"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "MqqyD0nP1LQ")}
/>
</div>
<div>
<div className="yt-clip-label">Music video</div>
<YouTube
videoId="kJQP7kiw5Fk"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "kJQP7kiw5Fk")}
/>
</div>
<div>
<div className="yt-clip-label">Politics</div>
<YouTube
videoId="F2b-2YnfZso"
opts={{
height: '90',
width: '150',
}}
onPlay={this.onSamplePlay.bind(this, "F2b-2YnfZso")}
/>
</div>
</div>
</div>
)}
{state === App.appStates.PROCESSING && (
{state === App.appStates.SAMPLE_PROCESSING && (
<YouTube
videoId={videoId}
opts={{
height: '390',
width: '640',
playerVars: {
autoplay: 1
}
}}
/>
)}
{isProcessing && (
<BarLoader
color={"rgb(54, 215, 183)"}
className={css`
margin: 0 auto;
margin: 20px auto 0 auto;
`}
width={10}
widthUnit="rem"

9
frontend/client/webpack.config.js
View file

@ -1,7 +1,8 @@
const merge = require("webpack-merge");
const UglifyJsPlugin = require("uglifyjs-webpack-plugin");
const webpack = require("webpack");
module.exports = env => {
const merge = require("webpack-merge");
const UglifyJsPlugin = require("uglifyjs-webpack-plugin");
const webpack = require("webpack");
let common = {
entry: {
@ -35,7 +36,7 @@ module.exports = env => {
}
};
if (env.production) {
if (env && env.production) {
return merge(common, {
plugins: [
new webpack.DefinePlugin({

16
frontend/static/frontend/css/app.css
View file

@ -61,7 +61,7 @@ img {
background-color: #eee;
}
.yt-label {
.yt-label, .yt-clips-label {
margin: 10px 0 10px 20px;
display: block;
}
@ -111,6 +111,20 @@ button:focus {
outline: 0;
}
.youtube-clips {
display: flex;
justify-content: space-between;
flex-wrap: wrap;
}
.youtube-clips > div {
}
.yt-clip-label {
}
@media all and (max-width: 700px) {
.wrap {
margin-top: 0;

14
frontend/static/frontend/js/app.client.js
View file

File diff suppressed because one or more lines are too long

32
keyframes/keyframes.py
View file

@ -25,25 +25,34 @@ logger = logging.getLogger(__name__)
class KeyFramesExtractor:
@classmethod
def get_keyframes(cls, video, gpu=settings.GPU, features_batch_size=settings.FEATURE_BATCH_SIZE):
frames_paths, all_frames_tmp_dir = cls._get_all_frames(video)
def get_keyframes(cls, video, gpu=settings.GPU, features_batch_size=settings.FEATURE_BATCH_SIZE,
frames_mode=0, rl_mode=0):
frames_paths, all_frames_tmp_dir = cls._get_all_frames(video, mode=frames_mode)
frames = cls._get_frames(frames_paths)
features = cls._get_features(frames, gpu, features_batch_size)
change_points, frames_per_segment = cls._get_segments(features)
probs = cls._get_probs(features, gpu)
probs = cls._get_probs(features, gpu, mode=rl_mode)
chosen_frames = cls._get_chosen_frames(frames, probs, change_points, frames_per_segment)
return chosen_frames
@staticmethod
def _get_all_frames(video):
def _get_all_frames(video, mode=0):
all_frames_tmp_dir = uuid.uuid4().hex
os.mkdir(jj(settings.TMP_DIR, all_frames_tmp_dir))
call(["ffmpeg", "-i", video.file.path, "-vf", "select=not(mod(n\\,15))", "-vsync", "vfr", "-q:v", "2",
jj(settings.TMP_DIR, all_frames_tmp_dir, "%06d.jpeg")])
if mode == 1:
call(["ffmpeg", "-i", f"{video.file.path}", "-c:v", "libxvid", "-qscale:v", "1", "-an",
jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "video.mp4")])
call(["ffmpeg", "-i", jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "video.mp4"), "-vf",
"select=eq(pict_type\,I)", "-vsync", "vfr",
jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "%06d.jpeg")])
else:
call(["ffmpeg", "-i", video.file.path, "-vf", "select=not(mod(n\\,15))", "-vsync", "vfr", "-q:v", "2",
jj(settings.TMP_DIR, all_frames_tmp_dir, "%06d.jpeg")])
frames_paths = []
for dirname, dirnames, filenames in os.walk(jj(settings.TMP_DIR, all_frames_tmp_dir)):
for filename in filenames:
frames_paths.append(jj(dirname, filename))
if not filename.endswith(".mp4"):
frames_paths.append(jj(dirname, filename))
return sorted(frames_paths), all_frames_tmp_dir
@staticmethod
@ -92,12 +101,15 @@ class KeyFramesExtractor:
return features.astype(np.float32)
@staticmethod
def _get_probs(features, gpu=True):
model_cache_key = "keyframes_rl_model_cache"
def _get_probs(features, gpu=True, mode=0):
model_cache_key = "keyframes_rl_model_cache_" + str(mode)
model = cache.get(model_cache_key) # get model from cache
if model is None:
model_path = "keyframes_rl/pretrained_model/model_epoch100.pth.tar"
if mode == 1:
model_path = "keyframes_rl/pretrained_model/model_1.pth.tar"
else:
model_path = "keyframes_rl/pretrained_model/model_0.pth.tar"
model = DSN(in_dim=1024, hid_dim=256, num_layers=1, cell="lstm")
if gpu:
checkpoint = torch.load(model_path)

BIN
keyframes_rl/pretrained_model/model_0.pth.tar Normal file
View file

Binary file not shown.

BIN
keyframes_rl/pretrained_model/model_epoch100.pth.tar → keyframes_rl/pretrained_model/model_1.pth.tar
View file

Binary file not shown.

40
package-lock.json generated
View file

@ -3953,6 +3953,11 @@
"invert-kv": "1.0.0"
}
},
"load-script": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/load-script/-/load-script-1.0.0.tgz",
"integrity": "sha1-BJGTngvuVkPuSUp+PaPSuscMbKQ="
},
"loader-runner": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.3.1.tgz",
@ -5271,6 +5276,16 @@
"recompose": "0.27.1"
}
},
"react-youtube": {
"version": "7.7.0",
"resolved": "https://registry.npmjs.org/react-youtube/-/react-youtube-7.7.0.tgz",
"integrity": "sha512-ZHHG3x7y9P8oCldPx0t4z0jTK9GC4lBVzKnYcd8SHQe2x5mCUIxLNWXzR8+Oe7H2I4ACCB87lLF5WuU39PGuCw==",
"requires": {
"lodash": "4.17.10",
"prop-types": "15.6.2",
"youtube-player": "5.5.1"
}
},
"readable-stream": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
@ -5629,6 +5644,11 @@
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
"dev": true
},
"sister": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/sister/-/sister-3.0.1.tgz",
"integrity": "sha512-aG41gNRHRRxPq52MpX4vtm9tapnr6ENmHUx8LMAJWCOplEMwXzh/dp5WIo52Wl8Zlc/VUyHLJ2snX0ck+Nma9g=="
},
"slash": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
@ -6637,6 +6657,26 @@
"requires": {
"camelcase": "4.1.0"
}
},
"youtube-player": {
"version": "5.5.1",
"resolved": "https://registry.npmjs.org/youtube-player/-/youtube-player-5.5.1.tgz",
"integrity": "sha512-1d62W9She0B1uKNyY6K7jtWFbOW3dYsm6hyKzrh11BLOuYFzkt8K6AcQ3QdPF3aU47dzhZ82clzOJVVWus4HTw==",
"requires": {
"debug": "2.6.9",
"load-script": "1.0.0",
"sister": "3.0.1"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
}
}
}
}
}

1
package.json
View file

@ -34,6 +34,7 @@
"react-dom": "^16.4.2",
"react-dropzone": "^5.0.1",
"react-spinners": "^0.4.5",
"react-youtube": "^7.7.0",
"style-loader": "^0.23.0",
"uglifyjs-webpack-plugin": "^1.2.5"
},

2
settings/settings.py
View file

@ -142,3 +142,5 @@ TMP_DIR = 'tmp/'
GPU = True
FEATURE_BATCH_SIZE = 32
DEFAULT_FRAMES_SAMPLING_MODE = 0
DEFAULT_RL_MODE = 0