diff --git a/ComixGAN/model.py b/ComixGAN/model.py
new file mode 100644
index 0000000..ec3439f
--- /dev/null
+++ b/ComixGAN/model.py
@@ -0,0 +1,24 @@
+import errno
+import os
+
+import tensorflow as tf
+from django.conf import settings
+from keras.models import load_model
+from keras_contrib.layers import InstanceNormalization
+
+
+class ComixGAN:
+ def __init__(self):
+ if not os.path.exists(settings.COMIX_GAN_MODEL_PATH):
+ raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), settings.COMIX_GAN_MODEL_PATH)
+ self.graph = tf.Graph()
+ config = tf.ConfigProto()
+ config.gpu_options.per_process_gpu_memory_fraction = 0.6
+ config.gpu_options.allow_growth = True
+ self.session = tf.Session(graph=self.graph, config=config)
+ with self.graph.as_default():
+ with self.session.as_default():
+ with tf.device('/device:GPU:0'):
+ self.model = load_model(settings.COMIX_GAN_MODEL_PATH,
+ custom_objects={'InstanceNormalization': InstanceNormalization})
+
diff --git a/ComixGAN/pretrained_models/generator_model.h5 b/ComixGAN/pretrained_models/generator_model.h5
new file mode 100644
index 0000000..48c80dc
Binary files /dev/null and b/ComixGAN/pretrained_models/generator_model.h5 differ
diff --git a/ComixGAN/pretrained_models/generator_model2.h5 b/ComixGAN/pretrained_models/generator_model2.h5
new file mode 100644
index 0000000..f5ca621
Binary files /dev/null and b/ComixGAN/pretrained_models/generator_model2.h5 differ
diff --git a/api/models.py b/api/models.py
index ee81408..5578d62 100644
--- a/api/models.py
+++ b/api/models.py
@@ -23,7 +23,7 @@ class Video(models.Model):
yt_pafy = pafy.new(yt_url)
# Use the biggest possible quality with file size < MAX_FILE_SIZE and resolution <= 480px
- for stream in yt_pafy.videostreams:
+ for stream in reversed(yt_pafy.videostreams):
if stream.get_filesize() < settings.MAX_FILE_SIZE and int(stream.quality.split("x")[1]) <= 480:
tmp_name = uuid.uuid4().hex + ".mp4"
relative_path = jj('raw_videos', tmp_name)
@@ -34,22 +34,24 @@ class Video(models.Model):
else:
raise TooLargeFile()
- def create_comic(self, frames_mode=0, rl_mode=0, image_assessment_mode=0):
+ def create_comic(self, frames_mode=0, rl_mode=0, image_assessment_mode=0, style_transfer_mode=0):
(keyframes, keyframes_timings), keyframes_extraction_time = KeyFramesExtractor.get_keyframes(
video=self,
frames_mode=frames_mode,
rl_mode=rl_mode,
image_assessment_mode=image_assessment_mode
)
- stylized_keyframes, stylization_time = StyleTransfer.get_stylized_frames(frames=keyframes)
+ stylized_keyframes, stylization_time = StyleTransfer.get_stylized_frames(frames=keyframes,
+ style_transfer_mode=style_transfer_mode)
comic_image, layout_generation_time = LayoutGenerator.get_layout(frames=stylized_keyframes)
timings = {
'keyframes_extraction_time': keyframes_extraction_time,
'stylization_time': stylization_time,
'layout_generation_time': layout_generation_time,
- **keyframes_timings
+ 'keyframes_extraction_time_details': keyframes_timings
}
+
return comic_image, timings
@@ -61,7 +63,7 @@ class Comic(models.Model):
@profile
def create_from_nparray(cls, nparray_file, video):
if nparray_file.max() <= 1:
- nparray_file = (nparray_file * 255).astype(int)
+ nparray_file = (nparray_file).astype(int)
tmp_name = uuid.uuid4().hex + ".png"
cv2.imwrite(jj(settings.TMP_DIR, tmp_name), nparray_file)
with open(jj(settings.TMP_DIR, tmp_name), mode="rb") as tmp_file:
diff --git a/api/serializers.py b/api/serializers.py
index 47c7c05..5510527 100644
--- a/api/serializers.py
+++ b/api/serializers.py
@@ -8,7 +8,10 @@ class VideoSerializer(serializers.Serializer):
file = serializers.FileField()
frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)
- image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE)
+ image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1,
+ default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE)
+ style_transfer_mode = serializers.IntegerField(min_value=0, max_value=2,
+ default=settings.DEFAULT_STYLE_TRANSFER_MODE)
def validate(self, attrs):
file = attrs.get("file")
@@ -23,4 +26,7 @@ class YouTubeDownloadSerializer(serializers.Serializer):
url = serializers.URLField()
frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE)
rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE)
- image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE)
+ image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1,
+ default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE)
+ style_transfer_mode = serializers.IntegerField(min_value=0, max_value=2,
+ default=settings.DEFAULT_STYLE_TRANSFER_MODE)
diff --git a/api/views.py b/api/views.py
index 77cde9d..9d6fdbe 100644
--- a/api/views.py
+++ b/api/views.py
@@ -22,7 +22,8 @@ class Comixify(APIView):
comic_image, timings = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"],
- image_assessment_mode=serializer.validated_data["image_assessment_mode"]
+ image_assessment_mode=serializer.validated_data["image_assessment_mode"],
+ style_transfer_mode=serializer.validated_data["style_transfer_mode"],
)
comic, from_nparray_time = Comic.create_from_nparray(comic_image, video)
timings['from_nparray_time'] = from_nparray_time
@@ -54,7 +55,8 @@ class ComixifyFromYoutube(APIView):
comic_image, timings = video.create_comic(
frames_mode=serializer.validated_data["frames_mode"],
rl_mode=serializer.validated_data["rl_mode"],
- image_assessment_mode=serializer.validated_data["image_assessment_mode"]
+ image_assessment_mode=serializer.validated_data["image_assessment_mode"],
+ style_transfer_mode=serializer.validated_data["style_transfer_mode"],
)
comic, from_nparray_time = Comic.create_from_nparray(comic_image, video)
timings['from_nparray_time'] = from_nparray_time
diff --git a/comic_layout/comic_layout.py b/comic_layout/comic_layout.py
index 27a9dad..07a0f9a 100644
--- a/comic_layout/comic_layout.py
+++ b/comic_layout/comic_layout.py
@@ -28,6 +28,6 @@ class LayoutGenerator():
def _pad_images(frames):
padded_result_imgs = []
for img in frames:
- padded_img = cv2.copyMakeBorder(img, 5, 5, 5, 5, cv2.BORDER_CONSTANT, value=(1, 1, 1))
+ padded_img = cv2.copyMakeBorder(img, 5, 5, 5, 5, cv2.BORDER_CONSTANT, value=(255, 255, 255))
padded_result_imgs.append(padded_img)
return padded_result_imgs
diff --git a/dockerfile b/dockerfile
index 718986e..4cb1229 100644
--- a/dockerfile
+++ b/dockerfile
@@ -10,10 +10,7 @@ RUN apt-get update && apt-get install -y apt-utils software-properties-common &&
libsnappy-dev protobuf-compiler \
python-numpy python-setuptools python-scipy \
libavformat-dev libswscale-dev unzip && \
- python3.6 -m pip install --upgrade pip && \
- python3.6 -m pip install jupyter ipywidgets jupyterlab && \
- python3.6 -m pip install h5py keras && \
- python3.6 -m pip install scikit-image opencv-contrib-python pyyaml
+ python3.6 -m pip install --upgrade pip
RUN mkdir /comixify
COPY ./Makefile.config /comixify/Makefile.config
@@ -45,7 +42,8 @@ RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig &&
WORKDIR /comixify
COPY . /comixify
RUN unzip popularity/pretrained_model/svr_test_11.10.sk.zip -d popularity/pretrained_model/ && \
- python3.6 -m pip install -r requirements.txt
+ python3.6 -m pip install -r requirements.txt && \
+ python3.6 -m pip install git+https://www.github.com/keras-team/keras-contrib.git
# Port to expose
diff --git a/frontend/client/App.js b/frontend/client/App.js
index db415a3..6890700 100644
--- a/frontend/client/App.js
+++ b/frontend/client/App.js
@@ -32,7 +32,8 @@ class App extends React.Component {
result_comics: null,
framesMode: "0",
rlMode: "0",
- imageAssessment: "0"
+ imageAssessment: "0",
+ styleTransferMode: "0",
};
this.onVideoDrop = this.onVideoDrop.bind(this);
this.onModelChange = this.onModelChange.bind(this);
@@ -40,6 +41,7 @@ class App extends React.Component {
this.onYouTubeSubmit = this.onYouTubeSubmit.bind(this);
this.onSamplingChange = this.onSamplingChange.bind(this);
this.onImageAssessmentChange = this.onImageAssessmentChange.bind(this);
+ this.styleTransferChange = this.styleTransferChange.bind(this);
}
static onVideoUploadProgress(progressEvent) {
let percentCompleted = Math.round(
@@ -52,6 +54,12 @@ class App extends React.Component {
this.setState({
rlMode: value
})
+ }
+ styleTransferChange(e) {
+ let value = e.currentTarget.value;
+ this.setState({
+ styleTransferMode: value
+ })
}
onSamplingChange(e) {
let value = e.currentTarget.value;
@@ -78,12 +86,13 @@ class App extends React.Component {
}
}
processVideo(video) {
- let { framesMode, rlMode, imageAssessment } = this.state;
+ let { framesMode, rlMode, imageAssessment, styleTransferMode } = this.state;
let data = new FormData();
data.append("file", video);
data.set('frames_mode', parseInt(framesMode));
data.set('rl_mode', parseInt(rlMode));
data.set("image_assessment_mode", parseInt(imageAssessment));
+ data.set('style_transfer_mode', parseInt(styleTransferMode));
post(COMIXIFY_API, data, {
headers: { "content-type": "multipart/form-data" },
onUploadProgress: App.onVideoUploadProgress
@@ -111,12 +120,13 @@ class App extends React.Component {
this.processVideo(files[0]);
}
submitYouTube(link) {
- let { framesMode, rlMode, imageAssessment } = this.state;
+ let { framesMode, rlMode, imageAssessment, styleTransferMode } = this.state;
post(FROM_YOUTUBE_API, {
url: link,
frames_mode: parseInt(framesMode),
rl_mode: parseInt(rlMode),
- image_assessment_mode: parseInt(imageAssessment)
+ image_assessment_mode: parseInt(imageAssessment),
+ style_transfer_mode: parseInt(styleTransferMode)
})
.then(this.handleResponse)
.catch(err => {
@@ -143,7 +153,7 @@ class App extends React.Component {
}
render() {
let {
- state, drop_errors, result_comics, framesMode, rlMode, videoId, imageAssessment
+ state, drop_errors, result_comics, framesMode, rlMode, videoId, imageAssessment, styleTransferMode
} = this.state;
let showUsage = [
App.appStates.INITIAL,
@@ -231,6 +241,36 @@ class App extends React.Component {
onChange={this.onImageAssessmentChange}
/>
+
+
+ Style Transfer model:
+
+
+
+
+
+
)}
diff --git a/frontend/static/frontend/js/app.client.js b/frontend/static/frontend/js/app.client.js
index ddd1f75..3be7132 100644
--- a/frontend/static/frontend/js/app.client.js
+++ b/frontend/static/frontend/js/app.client.js
@@ -1,9 +1,9 @@
-!function(e){var t={};function n(r){if(t[r])return t[r].exports;var o=t[r]={i:r,l:!1,exports:{}};return e[r].call(o.exports,o,o.exports,n),o.l=!0,o.exports}n.m=e,n.c=t,n.d=function(e,t,r){n.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:r})},n.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},n.t=function(e,t){if(1&t&&(e=n(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var o in e)n.d(r,o,function(t){return e[t]}.bind(null,o));return r},n.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return n.d(t,"a",t),t},n.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},n.p="",n(n.s=267)}([function(e,t,n){"use strict";e.exports=n(100)},function(e,t,n){e.exports=n(107)()},function(e,t,n){"use strict";n.r(t),function(e){n.d(t,"flush",function(){return i}),n.d(t,"hydrate",function(){return u}),n.d(t,"cx",function(){return l}),n.d(t,"merge",function(){return s}),n.d(t,"getRegisteredStyles",function(){return c}),n.d(t,"injectGlobal",function(){return f}),n.d(t,"keyframes",function(){return p}),n.d(t,"css",function(){return d}),n.d(t,"sheet",function(){return h}),n.d(t,"caches",function(){return m});var r=n(99),o=void 0!==e?e:{},a=Object(r.a)(o),i=a.flush,u=a.hydrate,l=a.cx,s=a.merge,c=a.getRegisteredStyles,f=a.injectGlobal,p=a.keyframes,d=a.css,h=a.sheet,m=a.caches}.call(this,n(41))},function(e,t,n){"use strict";t.__esModule=!0;var r=function(e){return e&&e.__esModule?e:{default:e}}(n(87));t.default=function(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!==(void 0===t?"undefined":(0,r.default)(t))&&"function"!=typeof t?e:t}},function(e,t,n){"use strict";n.r(t);var r=n(0),o=n.n(r),a=n(8),i=n.n(a),u=n(13),l=n.n(u),s=n(5),c=n.n(s),f=n(3),p=n.n(f),d=n(6),h=n.n(d);function m(){var e=this.constructor.getDerivedStateFromProps(this.props,this.state);null!==e&&void 0!==e&&this.setState(e)}function y(e){this.setState(function(t){var n=this.constructor.getDerivedStateFromProps(e,t);return null!==n&&void 0!==n?n:null}.bind(this))}function v(e,t){try{var n=this.props,r=this.state;this.props=e,this.state=t,this.__reactInternalSnapshotFlag=!0,this.__reactInternalSnapshot=this.getSnapshotBeforeUpdate(n,r)}finally{this.props=n,this.state=r}}function g(e){var t=e.prototype;if(!t||!t.isReactComponent)throw new Error("Can only polyfill class components");if("function"!=typeof e.getDerivedStateFromProps&&"function"!=typeof t.getSnapshotBeforeUpdate)return e;var n=null,r=null,o=null;if("function"==typeof t.componentWillMount?n="componentWillMount":"function"==typeof t.UNSAFE_componentWillMount&&(n="UNSAFE_componentWillMount"),"function"==typeof t.componentWillReceiveProps?r="componentWillReceiveProps":"function"==typeof t.UNSAFE_componentWillReceiveProps&&(r="UNSAFE_componentWillReceiveProps"),"function"==typeof t.componentWillUpdate?o="componentWillUpdate":"function"==typeof t.UNSAFE_componentWillUpdate&&(o="UNSAFE_componentWillUpdate"),null!==n||null!==r||null!==o){var a=e.displayName||e.name,i="function"==typeof e.getDerivedStateFromProps?"getDerivedStateFromProps()":"getSnapshotBeforeUpdate()";throw Error("Unsafe legacy lifecycles will not be called for components using new component APIs.\n\n"+a+" uses "+i+" but also contains the following legacy lifecycles:"+(null!==n?"\n "+n:"")+(null!==r?"\n "+r:"")+(null!==o?"\n "+o:"")+"\n\nThe above lifecycles should be removed. Learn more about this warning here:\nhttps://fb.me/react-async-component-lifecycle-hooks")}if("function"==typeof e.getDerivedStateFromProps&&(t.componentWillMount=m,t.componentWillReceiveProps=y),"function"==typeof t.getSnapshotBeforeUpdate){if("function"!=typeof t.componentDidUpdate)throw new Error("Cannot polyfill getSnapshotBeforeUpdate() for components that do not define componentDidUpdate() on the prototype");t.componentWillUpdate=v;var u=t.componentDidUpdate;t.componentDidUpdate=function(e,t,n){var r=this.__reactInternalSnapshotFlag?this.__reactInternalSnapshot:n;u.call(this,e,t,r)}}return e}m.__suppressDeprecationWarning=!0,y.__suppressDeprecationWarning=!0,v.__suppressDeprecationWarning=!0;var b=n(58),w=n.n(b),x=n(37),_=n.n(x),k=n(97),E=n.n(k),S=n(59),P=n(38);n.d(t,"mapProps",function(){return j}),n.d(t,"withProps",function(){return A}),n.d(t,"withPropsOnChange",function(){return z}),n.d(t,"withHandlers",function(){return D}),n.d(t,"defaultProps",function(){return M}),n.d(t,"renameProp",function(){return I}),n.d(t,"renameProps",function(){return V}),n.d(t,"flattenProp",function(){return B}),n.d(t,"withState",function(){return W}),n.d(t,"withStateHandlers",function(){return H}),n.d(t,"withReducer",function(){return q}),n.d(t,"branch",function(){return K}),n.d(t,"renderComponent",function(){return G}),n.d(t,"renderNothing",function(){return Q}),n.d(t,"shouldUpdate",function(){return Z}),n.d(t,"pure",function(){return J}),n.d(t,"onlyUpdateForKeys",function(){return ee}),n.d(t,"onlyUpdateForPropTypes",function(){return te}),n.d(t,"withContext",function(){return ne}),n.d(t,"getContext",function(){return re}),n.d(t,"lifecycle",function(){return oe}),n.d(t,"toClass",function(){return ie}),n.d(t,"withRenderProps",function(){return ue}),n.d(t,"setStatic",function(){return C}),n.d(t,"setPropTypes",function(){return le}),n.d(t,"setDisplayName",function(){return O}),n.d(t,"compose",function(){return se}),n.d(t,"getDisplayName",function(){return T}),n.d(t,"wrapDisplayName",function(){return N}),n.d(t,"isClassComponent",function(){return ae}),n.d(t,"createSink",function(){return ce}),n.d(t,"componentFromProp",function(){return fe}),n.d(t,"nest",function(){return pe}),n.d(t,"hoistStatics",function(){return de}),n.d(t,"componentFromStream",function(){return ge}),n.d(t,"componentFromStreamWithConfig",function(){return ve}),n.d(t,"mapPropsStream",function(){return xe}),n.d(t,"mapPropsStreamWithConfig",function(){return we}),n.d(t,"createEventHandler",function(){return ke}),n.d(t,"createEventHandlerWithConfig",function(){return _e}),n.d(t,"setObservableConfig",function(){return me}),n.d(t,"shallowEqual",function(){return l.a});var C=function(e,t){return function(n){return n[e]=t,n}},O=function(e){return C("displayName",e)},T=function(e){return"string"==typeof e?e:e?e.displayName||e.name||"Component":void 0},N=function(e,t){return t+"("+T(e)+")"},j=function(e){return function(t){var n=Object(r.createFactory)(t);return function(t){return n(e(t))}}},A=function(e){return j(function(t){return i()({},t,"function"==typeof e?e(t):e)})},U=function(e,t){for(var n={},r=0;r1?r-1:0),a=1;a1&&void 0!==arguments[1]?arguments[1]:$;return t.setState(function(t){var r=t.stateValue;return{stateValue:n(r,e)}},function(){return r(t.state.stateValue)})},o=e,p()(t,o)}return h()(a,r),a.prototype.initializeStateValue=function(){return void 0!==o?"function"==typeof o?o(this.props):o:n(void 0,{type:"@@recompose/INIT"})},a.prototype.render=function(){var n;return u(i()({},this.props,((n={})[e]=this.state.stateValue,n[t]=this.dispatch,n)))},a}(r.Component)}},Y=function(e){return e},K=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:Y;return function(o){var a=void 0,i=void 0;return function(u){return e(u)?(a=a||Object(r.createFactory)(t(o)))(u):(i=i||Object(r.createFactory)(n(o)))(u)}}},G=function(e){return function(t){var n=Object(r.createFactory)(e);return function(e){return n(e)}}},X=function(e){function t(){return c()(this,t),p()(this,e.apply(this,arguments))}return h()(t,e),t.prototype.render=function(){return null},t}(r.Component),Q=function(e){return X},Z=function(e){return function(t){var n=Object(r.createFactory)(t);return function(t){function r(){return c()(this,r),p()(this,t.apply(this,arguments))}return h()(r,t),r.prototype.shouldComponentUpdate=function(t){return e(this.props,t)},r.prototype.render=function(){return n(this.props)},r}(r.Component)}},J=function(e){return Z(function(e,t){return!l()(e,t)})(e)},ee=function(e){return Z(function(t,n){return!l()(U(n,e),U(t,e))})},te=function(e){var t=e.propTypes,n=_()(t||{});return ee(n)(e)},ne=function(e,t){return function(n){var o=Object(r.createFactory)(n),a=function(e){function n(){var r,o,a;c()(this,n);for(var i=arguments.length,u=Array(i),l=0;l=200&&e<300},headers:{common:{Accept:"application/json, text/plain, */*"}}};r.forEach(["delete","get","head"],function(e){u.headers[e]={}}),r.forEach(["post","put","patch"],function(e){u.headers[e]=r.merge(a)}),e.exports=u}).call(this,n(75))},function(e,t,n){var r=n(18);e.exports=function(e,t){if(!r(e))return e;var n,o;if(t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;if("function"==typeof(n=e.valueOf)&&!r(o=n.call(e)))return o;if(!t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;throw TypeError("Can't convert object to primitive value")}},function(e,t){e.exports=function(e){if(void 0==e)throw TypeError("Can't call method on "+e);return e}},function(e,t){var n=Math.ceil,r=Math.floor;e.exports=function(e){return isNaN(e=+e)?0:(e>0?r:n)(e)}},function(e,t,n){var r=n(48)("keys"),o=n(34);e.exports=function(e){return r[e]||(r[e]=o(e))}},function(e,t,n){var r=n(10),o=n(11),a=o["__core-js_shared__"]||(o["__core-js_shared__"]={});(e.exports=function(e,t){return a[e]||(a[e]=void 0!==t?t:{})})("versions",[]).push({version:r.version,mode:n(33)?"pure":"global",copyright:"© 2018 Denis Pushkarev (zloirock.ru)"})},function(e,t){e.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(e,t){t.f=Object.getOwnPropertySymbols},function(e,t,n){var r=n(45);e.exports=function(e){return Object(r(e))}},function(e,t){e.exports={}},function(e,t,n){var r=n(24),o=n(221),a=n(49),i=n(47)("IE_PROTO"),u=function(){},l=function(){var e,t=n(83)("iframe"),r=a.length;for(t.style.display="none",n(222).appendChild(t),t.src="javascript:",(e=t.contentWindow.document).open(),e.write("