Add pipeline selection

Change model files

Add frontend

Increase quality of I-frame selector

Add YouTube clips

Move defaults to settings

Fix Sports video

api/models.py

@@ -33,8 +33,8 @@ class Video(models.Model):
         else:
             raise TooLargeFile()
 
-    def create_comic(self):
+    def create_comic(self, frames_mode=0, rl_mode=0):
-        keyframes = KeyFramesExtractor.get_keyframes(video=self)
+        keyframes = KeyFramesExtractor.get_keyframes(video=self, frames_mode=frames_mode, rl_mode=rl_mode)
         stylized_keyframes = StyleTransfer.get_stylized_frames(frames=keyframes)
         comic_image = LayoutGenerator.get_layout(frames=stylized_keyframes)
         return comic_image

api/serializers.py

@@ -2,13 +2,12 @@ from django.conf import settings
 from rest_framework import serializers
 
 from .exceptions import FileExtensionError, TooLargeFile
-from .models import Video
 
 
-class VideoSerializer(serializers.ModelSerializer):
+class VideoSerializer(serializers.Serializer):
-    class Meta:
+    file = serializers.FileField()
-        model = Video
+    frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
-        fields = ("file", "timestamp")
+    rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)
 
     def validate(self, attrs):
         file = attrs.get("file")
@@ -21,3 +20,5 @@ class VideoSerializer(serializers.ModelSerializer):
 
 class YouTubeDownloadSerializer(serializers.Serializer):
     url = serializers.URLField()
+    frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
+    rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)

api/views.py

@@ -19,7 +19,10 @@ class Comixify(APIView):
 
         video_file = serializer.validated_data["file"]
         video = Video.objects.create(file=video_file)
-        comic_image = video.create_comic()
+        comic_image = video.create_comic(
+            frames_mode=serializer.validated_data["frames_mode"],
+            rl_mode=serializer.validated_data["rl_mode"]
+        )
         comic = Comic.create_from_nparray(comic_image, video)
 
         response = {
@@ -45,7 +48,10 @@ class ComixifyFromYoutube(APIView):
         video = Video()
         video.download_from_youtube(yt_url)
         video.save()
-        comic_image = video.create_comic()
+        comic_image = video.create_comic(
+            frames_mode=serializer.validated_data["frames_mode"],
+            rl_mode=serializer.validated_data["rl_mode"]
+        )
         comic = Comic.create_from_nparray(comic_image, video)
 
         response = {

frontend/client/App.js

@@ -1,5 +1,6 @@
 import React from "react";
 import ReactDOM from "react-dom";
+import YouTube from 'react-youtube';
 import { post } from "axios";
 import Dropzone from "react-dropzone";
 import { BarLoader } from "react-spinners";
@@ -18,27 +19,44 @@ class App extends React.Component {
 		PROCESSING: 1,
 		FINISHED: 2,
 		UPLOAD_ERROR: 3,
-		DROP_ERROR: 4
+		DROP_ERROR: 4,
+		SAMPLE_PROCESSING: 5
 	};
 	ytInput = React.createRef();
 	constructor(props) {
 		super(props);
 		this.state = {
 			state: App.appStates.INITIAL,
+			videoId: null,
 			drop_errors: [],
-			result_comics: null
+			result_comics: null,
+            framesMode: "0",
+            rlMode: "0"
 		};
 		this.onVideoDrop = this.onVideoDrop.bind(this);
+        this.onModelChange = this.onModelChange.bind(this);
         this.handleResponse = this.handleResponse.bind(this);
         this.onYouTubeSubmit = this.onYouTubeSubmit.bind(this);
-		this.onVideoUploadProgress = this.onVideoUploadProgress.bind(this);
+        this.onSamplingChange = this.onSamplingChange.bind(this);
 	}
-	onVideoUploadProgress(progressEvent) {
+	static onVideoUploadProgress(progressEvent) {
 		let percentCompleted = Math.round(
 			progressEvent.loaded * 100 / progressEvent.total
 		);
 		console.log(percentCompleted);
 	}
+	onModelChange(e) {
+        let value = e.currentTarget.value;
+	    this.setState({
+            rlMode: value
+        })
+    }
+	onSamplingChange(e) {
+	    let value = e.currentTarget.value;
+	    this.setState({
+            framesMode: value
+        })
+    }
 	handleResponse(res) {
 	    if (res.data["status_message"] === "ok") {
             this.setState({
@@ -52,11 +70,14 @@ class App extends React.Component {
         }
     }
 	processVideo(video) {
+		let { framesMode, rlMode } = this.state
 		let data = new FormData();
 		data.append("file", video);
+		data.set('frames_mode', parseInt(framesMode));
+		data.set('rl_mode', parseInt(rlMode));
 		post(COMIXIFY_API, data, {
 			headers: { "content-type": "multipart/form-data" },
-			onUploadProgress: this.onVideoUploadProgress
+			onUploadProgress: App.onVideoUploadProgress
 		})
 			.then(this.handleResponse)
 			.catch(err => {
@@ -80,10 +101,12 @@ class App extends React.Component {
 		}
 		this.processVideo(files[0]);
 	}
-	onYouTubeSubmit() {
+	submitYouTube(link) {
-		let ytLink = this.ytInput.current.value;
+	    let { framesMode, rlMode } = this.state;
-		post(FROM_YOUTUBE_API, {
+	    post(FROM_YOUTUBE_API, {
-		    url: ytLink
+		    url: link,
+			frames_mode: parseInt(framesMode),
+			rl_mode: parseInt(rlMode)
         })
 			.then(this.handleResponse)
 			.catch(err => {
@@ -92,18 +115,34 @@ class App extends React.Component {
 					state: App.appStates.UPLOAD_ERROR
 				});
 			});
+    }
+	onYouTubeSubmit() {
+		let ytLink = this.ytInput.current.value;
+		this.submitYouTube(ytLink);
 		this.setState({
 			state: App.appStates.PROCESSING
 		});
 	}
+	onSamplePlay(videoId) {
+	    let link = "https://www.youtube.com/watch?v=" + videoId;
+	    this.submitYouTube(link);
+		this.setState({
+			videoId: videoId,
+			state: App.appStates.SAMPLE_PROCESSING
+		});
+	}
 	render() {
-		let { state, drop_errors, result_comics } = this.state;
+		let { state, drop_errors, result_comics, framesMode, rlMode, videoId } = this.state;
 		let showUsage = [
 			App.appStates.INITIAL,
 			App.appStates.UPLOAD_ERROR,
 			App.appStates.DROP_ERROR,
 			App.appStates.FINISHED
 		].includes(state);
+		let isProcessing = [
+			App.appStates.SAMPLE_PROCESSING,
+			App.appStates.PROCESSING
+		].includes(state);
 		return (
 			<div>
 				{state === App.appStates.FINISHED && [
@@ -115,6 +154,53 @@ class App extends React.Component {
 				{state === App.appStates.UPLOAD_ERROR && (
 					<p>Server Error: Please try again later.</p>
 				)}
+				{showUsage && (
+				    <div>
+                        <div>Pipeline settings:</div>
+                        <div>
+                            <span>Frame sampling:</span>
+                            <input
+                                type="radio"
+                                name="sampling"
+                                id="sampling-0"
+                                value="0"
+                                checked={framesMode === "0"}
+                                onChange={this.onSamplingChange}
+                            />
+                            <label htmlFor="sampling-0">2fps sampling</label>
+                            <input
+                                type="radio"
+                                name="sampling"
+                                id="sampling-1"
+                                value="1"
+                                checked={framesMode === "1"}
+                                onChange={this.onSamplingChange}
+                            />
+                            <label htmlFor="sampling-1">I-frame sampling</label>
+                        </div>
+                        <div>
+                            <span>Extraction model:</span>
+                            <input
+                                type="radio"
+                                name="model"
+                                id="model-0"
+                                value="0"
+                                checked={rlMode === "0"}
+                                onChange={this.onModelChange}
+                            />
+                            <label htmlFor="model-0">Basic model</label>
+                            <input
+                                type="radio"
+                                name="model"
+                                id="model-1"
+                                value="1"
+                                checked={rlMode === "1"}
+                                onChange={this.onModelChange}
+                            />
+                            <label htmlFor="model-1">+VTW model</label>
+                        </div>
+                    </div>
+                )}
 				{showUsage && (
 					<Dropzone
 						onDrop={this.onVideoDrop}
@@ -133,13 +219,72 @@ class App extends React.Component {
 						<label htmlFor="yt-link" className="yt-label">Or use YouTube link:</label>
 						<input type="url" id="yt-link" ref={this.ytInput}/>
 						<button onClick={this.onYouTubeSubmit}>Run</button>
+						<div className="yt-clips-label">Or select one of sample videos:</div>
+						<div className="youtube-clips">
+                            <div>
+                                <div className="yt-clip-label">Documentary</div>
+                                <YouTube
+                                    videoId="gr1ps0ooDhU"
+                                    opts={{
+                                        height: '90',
+                                        width: '150',
+                                    }}
+                                    onPlay={this.onSamplePlay.bind(this, "gr1ps0ooDhU")}
+                                />
+                            </div>
+                            <div>
+                                <div className="yt-clip-label">Sports</div>
+                                <YouTube
+                                    videoId="MqqyD0nP1LQ"
+                                    opts={{
+                                        height: '90',
+                                        width: '150',
+                                    }}
+                                    onPlay={this.onSamplePlay.bind(this, "MqqyD0nP1LQ")}
+                                />
+                            </div>
+                            <div>
+                                <div className="yt-clip-label">Music video</div>
+                                <YouTube
+                                    videoId="kJQP7kiw5Fk"
+                                    opts={{
+                                        height: '90',
+                                        width: '150',
+                                    }}
+                                    onPlay={this.onSamplePlay.bind(this, "kJQP7kiw5Fk")}
+                                />
+                            </div>
+                            <div>
+                                <div className="yt-clip-label">Politics</div>
+                                <YouTube
+                                    videoId="F2b-2YnfZso"
+                                    opts={{
+                                        height: '90',
+                                        width: '150',
+                                    }}
+                                    onPlay={this.onSamplePlay.bind(this, "F2b-2YnfZso")}
+                                />
+						    </div>
+						</div>
 					</div>
 				)}
-				{state === App.appStates.PROCESSING && (
+				{state === App.appStates.SAMPLE_PROCESSING && (
+					<YouTube
+						videoId={videoId}
+						opts={{
+							height: '390',
+							width: '640',
+							playerVars: {
+								autoplay: 1
+							}
+						}}
+					/>
+				)}
+				{isProcessing && (
 					<BarLoader
 						color={"rgb(54, 215, 183)"}
 						className={css`
-							margin: 0 auto;
+							margin: 20px auto 0 auto;
 						`}
 						width={10}
 						widthUnit="rem"

frontend/client/webpack.config.js

@@ -1,7 +1,8 @@
+const merge = require("webpack-merge");
+const UglifyJsPlugin = require("uglifyjs-webpack-plugin");
+const webpack = require("webpack");
+
 module.exports = env => {
-    const merge = require("webpack-merge");
-    const UglifyJsPlugin = require("uglifyjs-webpack-plugin");
-    const webpack = require("webpack");
 
     let common = {
         entry: {
@@ -35,7 +36,7 @@ module.exports = env => {
         }
     };
 
-    if (env.production) {
+    if (env && env.production) {
         return merge(common, {
             plugins: [
                 new webpack.DefinePlugin({

frontend/static/frontend/css/app.css

@@ -61,7 +61,7 @@ img {
 	background-color: #eee;
 }
 
-.yt-label {
+.yt-label, .yt-clips-label {
 	margin: 10px 0 10px 20px;
 	display: block;
 }
@@ -111,6 +111,20 @@ button:focus {
 	outline: 0;
 }
 
+.youtube-clips {
+	display: flex;
+	justify-content: space-between;
+	flex-wrap: wrap;
+}
+
+.youtube-clips > div {
+
+}
+
+.yt-clip-label {
+
+}
+
 @media all and (max-width: 700px) {
 	.wrap {
 		margin-top: 0;

keyframes/keyframes.py

@@ -25,25 +25,34 @@ logger = logging.getLogger(__name__)
 
 class KeyFramesExtractor:
     @classmethod
-    def get_keyframes(cls, video, gpu=settings.GPU, features_batch_size=settings.FEATURE_BATCH_SIZE):
+    def get_keyframes(cls, video, gpu=settings.GPU, features_batch_size=settings.FEATURE_BATCH_SIZE,
-        frames_paths, all_frames_tmp_dir = cls._get_all_frames(video)
+                      frames_mode=0, rl_mode=0):
+        frames_paths, all_frames_tmp_dir = cls._get_all_frames(video, mode=frames_mode)
         frames = cls._get_frames(frames_paths)
         features = cls._get_features(frames, gpu, features_batch_size)
         change_points, frames_per_segment = cls._get_segments(features)
-        probs = cls._get_probs(features, gpu)
+        probs = cls._get_probs(features, gpu, mode=rl_mode)
         chosen_frames = cls._get_chosen_frames(frames, probs, change_points, frames_per_segment)
         return chosen_frames
 
     @staticmethod
-    def _get_all_frames(video):
+    def _get_all_frames(video, mode=0):
         all_frames_tmp_dir = uuid.uuid4().hex
         os.mkdir(jj(settings.TMP_DIR, all_frames_tmp_dir))
-        call(["ffmpeg", "-i", video.file.path, "-vf", "select=not(mod(n\\,15))", "-vsync", "vfr", "-q:v", "2",
+        if mode == 1:
-              jj(settings.TMP_DIR, all_frames_tmp_dir, "%06d.jpeg")])
+            call(["ffmpeg", "-i", f"{video.file.path}", "-c:v", "libxvid", "-qscale:v", "1", "-an",
+                  jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "video.mp4")])
+            call(["ffmpeg", "-i", jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "video.mp4"), "-vf",
+                  "select=eq(pict_type\,I)", "-vsync", "vfr",
+                  jj(f"{settings.TMP_DIR}", f"{all_frames_tmp_dir}", "%06d.jpeg")])
+        else:
+            call(["ffmpeg", "-i", video.file.path, "-vf", "select=not(mod(n\\,15))", "-vsync", "vfr", "-q:v", "2",
+                  jj(settings.TMP_DIR, all_frames_tmp_dir, "%06d.jpeg")])
         frames_paths = []
         for dirname, dirnames, filenames in os.walk(jj(settings.TMP_DIR, all_frames_tmp_dir)):
             for filename in filenames:
-                frames_paths.append(jj(dirname, filename))
+                if not filename.endswith(".mp4"):
+                    frames_paths.append(jj(dirname, filename))
         return sorted(frames_paths), all_frames_tmp_dir
 
     @staticmethod
@@ -92,12 +101,15 @@ class KeyFramesExtractor:
         return features.astype(np.float32)
 
     @staticmethod
-    def _get_probs(features, gpu=True):
+    def _get_probs(features, gpu=True, mode=0):
-        model_cache_key = "keyframes_rl_model_cache"
+        model_cache_key = "keyframes_rl_model_cache_" + str(mode)
         model = cache.get(model_cache_key)  # get model from cache
 
         if model is None:
-            model_path = "keyframes_rl/pretrained_model/model_epoch100.pth.tar"
+            if mode == 1:
+                model_path = "keyframes_rl/pretrained_model/model_1.pth.tar"
+            else:
+                model_path = "keyframes_rl/pretrained_model/model_0.pth.tar"
             model = DSN(in_dim=1024, hid_dim=256, num_layers=1, cell="lstm")
             if gpu:
                 checkpoint = torch.load(model_path)

package-lock.json

@@ -3953,6 +3953,11 @@
         "invert-kv": "1.0.0"
       }
     },
+    "load-script": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/load-script/-/load-script-1.0.0.tgz",
+      "integrity": "sha1-BJGTngvuVkPuSUp+PaPSuscMbKQ="
+    },
     "loader-runner": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.3.1.tgz",
@@ -5271,6 +5276,16 @@
         "recompose": "0.27.1"
       }
     },
+    "react-youtube": {
+      "version": "7.7.0",
+      "resolved": "https://registry.npmjs.org/react-youtube/-/react-youtube-7.7.0.tgz",
+      "integrity": "sha512-ZHHG3x7y9P8oCldPx0t4z0jTK9GC4lBVzKnYcd8SHQe2x5mCUIxLNWXzR8+Oe7H2I4ACCB87lLF5WuU39PGuCw==",
+      "requires": {
+        "lodash": "4.17.10",
+        "prop-types": "15.6.2",
+        "youtube-player": "5.5.1"
+      }
+    },
     "readable-stream": {
       "version": "2.3.6",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
@@ -5629,6 +5644,11 @@
       "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
       "dev": true
     },
+    "sister": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/sister/-/sister-3.0.1.tgz",
+      "integrity": "sha512-aG41gNRHRRxPq52MpX4vtm9tapnr6ENmHUx8LMAJWCOplEMwXzh/dp5WIo52Wl8Zlc/VUyHLJ2snX0ck+Nma9g=="
+    },
     "slash": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
@@ -6637,6 +6657,26 @@
       "requires": {
         "camelcase": "4.1.0"
       }
+    },
+    "youtube-player": {
+      "version": "5.5.1",
+      "resolved": "https://registry.npmjs.org/youtube-player/-/youtube-player-5.5.1.tgz",
+      "integrity": "sha512-1d62W9She0B1uKNyY6K7jtWFbOW3dYsm6hyKzrh11BLOuYFzkt8K6AcQ3QdPF3aU47dzhZ82clzOJVVWus4HTw==",
+      "requires": {
+        "debug": "2.6.9",
+        "load-script": "1.0.0",
+        "sister": "3.0.1"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
     }
   }
 }

package.json

@@ -34,6 +34,7 @@
     "react-dom": "^16.4.2",
     "react-dropzone": "^5.0.1",
     "react-spinners": "^0.4.5",
+    "react-youtube": "^7.7.0",
     "style-loader": "^0.23.0",
     "uglifyjs-webpack-plugin": "^1.2.5"
   },

settings/settings.py

@@ -142,3 +142,5 @@ TMP_DIR = 'tmp/'
 GPU = True
 
 FEATURE_BATCH_SIZE = 32
+DEFAULT_FRAMES_SAMPLING_MODE = 0
+DEFAULT_RL_MODE = 0