diff --git a/ComixGAN/model.py b/ComixGAN/model.py new file mode 100644 index 0000000..ec3439f --- /dev/null +++ b/ComixGAN/model.py @@ -0,0 +1,24 @@ +import errno +import os + +import tensorflow as tf +from django.conf import settings +from keras.models import load_model +from keras_contrib.layers import InstanceNormalization + + +class ComixGAN: + def __init__(self): + if not os.path.exists(settings.COMIX_GAN_MODEL_PATH): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), settings.COMIX_GAN_MODEL_PATH) + self.graph = tf.Graph() + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.6 + config.gpu_options.allow_growth = True + self.session = tf.Session(graph=self.graph, config=config) + with self.graph.as_default(): + with self.session.as_default(): + with tf.device('/device:GPU:0'): + self.model = load_model(settings.COMIX_GAN_MODEL_PATH, + custom_objects={'InstanceNormalization': InstanceNormalization}) + diff --git a/ComixGAN/pretrained_models/generator_model.h5 b/ComixGAN/pretrained_models/generator_model.h5 new file mode 100644 index 0000000..48c80dc Binary files /dev/null and b/ComixGAN/pretrained_models/generator_model.h5 differ diff --git a/ComixGAN/pretrained_models/generator_model2.h5 b/ComixGAN/pretrained_models/generator_model2.h5 new file mode 100644 index 0000000..f5ca621 Binary files /dev/null and b/ComixGAN/pretrained_models/generator_model2.h5 differ diff --git a/api/models.py b/api/models.py index ee81408..5578d62 100644 --- a/api/models.py +++ b/api/models.py @@ -23,7 +23,7 @@ class Video(models.Model): yt_pafy = pafy.new(yt_url) # Use the biggest possible quality with file size < MAX_FILE_SIZE and resolution <= 480px - for stream in yt_pafy.videostreams: + for stream in reversed(yt_pafy.videostreams): if stream.get_filesize() < settings.MAX_FILE_SIZE and int(stream.quality.split("x")[1]) <= 480: tmp_name = uuid.uuid4().hex + ".mp4" relative_path = jj('raw_videos', tmp_name) @@ -34,22 +34,24 @@ class Video(models.Model): else: raise TooLargeFile() - def create_comic(self, frames_mode=0, rl_mode=0, image_assessment_mode=0): + def create_comic(self, frames_mode=0, rl_mode=0, image_assessment_mode=0, style_transfer_mode=0): (keyframes, keyframes_timings), keyframes_extraction_time = KeyFramesExtractor.get_keyframes( video=self, frames_mode=frames_mode, rl_mode=rl_mode, image_assessment_mode=image_assessment_mode ) - stylized_keyframes, stylization_time = StyleTransfer.get_stylized_frames(frames=keyframes) + stylized_keyframes, stylization_time = StyleTransfer.get_stylized_frames(frames=keyframes, + style_transfer_mode=style_transfer_mode) comic_image, layout_generation_time = LayoutGenerator.get_layout(frames=stylized_keyframes) timings = { 'keyframes_extraction_time': keyframes_extraction_time, 'stylization_time': stylization_time, 'layout_generation_time': layout_generation_time, - **keyframes_timings + 'keyframes_extraction_time_details': keyframes_timings } + return comic_image, timings @@ -61,7 +63,7 @@ class Comic(models.Model): @profile def create_from_nparray(cls, nparray_file, video): if nparray_file.max() <= 1: - nparray_file = (nparray_file * 255).astype(int) + nparray_file = (nparray_file).astype(int) tmp_name = uuid.uuid4().hex + ".png" cv2.imwrite(jj(settings.TMP_DIR, tmp_name), nparray_file) with open(jj(settings.TMP_DIR, tmp_name), mode="rb") as tmp_file: diff --git a/api/serializers.py b/api/serializers.py index 47c7c05..5510527 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -8,7 +8,10 @@ class VideoSerializer(serializers.Serializer): file = serializers.FileField() frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE) rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE) - image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE) + image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1, + default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE) + style_transfer_mode = serializers.IntegerField(min_value=0, max_value=2, + default=settings.DEFAULT_STYLE_TRANSFER_MODE) def validate(self, attrs): file = attrs.get("file") @@ -23,4 +26,7 @@ class YouTubeDownloadSerializer(serializers.Serializer): url = serializers.URLField() frames_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_FRAMES_SAMPLING_MODE) rl_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_RL_MODE) - image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1, default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE) + image_assessment_mode = serializers.IntegerField(min_value=0, max_value=1, + default=settings.DEFAULT_IMAGE_ASSESSMENT_MODE) + style_transfer_mode = serializers.IntegerField(min_value=0, max_value=2, + default=settings.DEFAULT_STYLE_TRANSFER_MODE) diff --git a/api/views.py b/api/views.py index 77cde9d..9d6fdbe 100644 --- a/api/views.py +++ b/api/views.py @@ -22,7 +22,8 @@ class Comixify(APIView): comic_image, timings = video.create_comic( frames_mode=serializer.validated_data["frames_mode"], rl_mode=serializer.validated_data["rl_mode"], - image_assessment_mode=serializer.validated_data["image_assessment_mode"] + image_assessment_mode=serializer.validated_data["image_assessment_mode"], + style_transfer_mode=serializer.validated_data["style_transfer_mode"], ) comic, from_nparray_time = Comic.create_from_nparray(comic_image, video) timings['from_nparray_time'] = from_nparray_time @@ -54,7 +55,8 @@ class ComixifyFromYoutube(APIView): comic_image, timings = video.create_comic( frames_mode=serializer.validated_data["frames_mode"], rl_mode=serializer.validated_data["rl_mode"], - image_assessment_mode=serializer.validated_data["image_assessment_mode"] + image_assessment_mode=serializer.validated_data["image_assessment_mode"], + style_transfer_mode=serializer.validated_data["style_transfer_mode"], ) comic, from_nparray_time = Comic.create_from_nparray(comic_image, video) timings['from_nparray_time'] = from_nparray_time diff --git a/comic_layout/comic_layout.py b/comic_layout/comic_layout.py index 27a9dad..07a0f9a 100644 --- a/comic_layout/comic_layout.py +++ b/comic_layout/comic_layout.py @@ -28,6 +28,6 @@ class LayoutGenerator(): def _pad_images(frames): padded_result_imgs = [] for img in frames: - padded_img = cv2.copyMakeBorder(img, 5, 5, 5, 5, cv2.BORDER_CONSTANT, value=(1, 1, 1)) + padded_img = cv2.copyMakeBorder(img, 5, 5, 5, 5, cv2.BORDER_CONSTANT, value=(255, 255, 255)) padded_result_imgs.append(padded_img) return padded_result_imgs diff --git a/dockerfile b/dockerfile index 718986e..4cb1229 100644 --- a/dockerfile +++ b/dockerfile @@ -10,10 +10,7 @@ RUN apt-get update && apt-get install -y apt-utils software-properties-common && libsnappy-dev protobuf-compiler \ python-numpy python-setuptools python-scipy \ libavformat-dev libswscale-dev unzip && \ - python3.6 -m pip install --upgrade pip && \ - python3.6 -m pip install jupyter ipywidgets jupyterlab && \ - python3.6 -m pip install h5py keras && \ - python3.6 -m pip install scikit-image opencv-contrib-python pyyaml + python3.6 -m pip install --upgrade pip RUN mkdir /comixify COPY ./Makefile.config /comixify/Makefile.config @@ -45,7 +42,8 @@ RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig && WORKDIR /comixify COPY . /comixify RUN unzip popularity/pretrained_model/svr_test_11.10.sk.zip -d popularity/pretrained_model/ && \ - python3.6 -m pip install -r requirements.txt + python3.6 -m pip install -r requirements.txt && \ + python3.6 -m pip install git+https://www.github.com/keras-team/keras-contrib.git # Port to expose diff --git a/frontend/client/App.js b/frontend/client/App.js index db415a3..6890700 100644 --- a/frontend/client/App.js +++ b/frontend/client/App.js @@ -32,7 +32,8 @@ class App extends React.Component { result_comics: null, framesMode: "0", rlMode: "0", - imageAssessment: "0" + imageAssessment: "0", + styleTransferMode: "0", }; this.onVideoDrop = this.onVideoDrop.bind(this); this.onModelChange = this.onModelChange.bind(this); @@ -40,6 +41,7 @@ class App extends React.Component { this.onYouTubeSubmit = this.onYouTubeSubmit.bind(this); this.onSamplingChange = this.onSamplingChange.bind(this); this.onImageAssessmentChange = this.onImageAssessmentChange.bind(this); + this.styleTransferChange = this.styleTransferChange.bind(this); } static onVideoUploadProgress(progressEvent) { let percentCompleted = Math.round( @@ -52,6 +54,12 @@ class App extends React.Component { this.setState({ rlMode: value }) + } + styleTransferChange(e) { + let value = e.currentTarget.value; + this.setState({ + styleTransferMode: value + }) } onSamplingChange(e) { let value = e.currentTarget.value; @@ -78,12 +86,13 @@ class App extends React.Component { } } processVideo(video) { - let { framesMode, rlMode, imageAssessment } = this.state; + let { framesMode, rlMode, imageAssessment, styleTransferMode } = this.state; let data = new FormData(); data.append("file", video); data.set('frames_mode', parseInt(framesMode)); data.set('rl_mode', parseInt(rlMode)); data.set("image_assessment_mode", parseInt(imageAssessment)); + data.set('style_transfer_mode', parseInt(styleTransferMode)); post(COMIXIFY_API, data, { headers: { "content-type": "multipart/form-data" }, onUploadProgress: App.onVideoUploadProgress @@ -111,12 +120,13 @@ class App extends React.Component { this.processVideo(files[0]); } submitYouTube(link) { - let { framesMode, rlMode, imageAssessment } = this.state; + let { framesMode, rlMode, imageAssessment, styleTransferMode } = this.state; post(FROM_YOUTUBE_API, { url: link, frames_mode: parseInt(framesMode), rl_mode: parseInt(rlMode), - image_assessment_mode: parseInt(imageAssessment) + image_assessment_mode: parseInt(imageAssessment), + style_transfer_mode: parseInt(styleTransferMode) }) .then(this.handleResponse) .catch(err => { @@ -143,7 +153,7 @@ class App extends React.Component { } render() { let { - state, drop_errors, result_comics, framesMode, rlMode, videoId, imageAssessment + state, drop_errors, result_comics, framesMode, rlMode, videoId, imageAssessment, styleTransferMode } = this.state; let showUsage = [ App.appStates.INITIAL, @@ -231,6 +241,36 @@ class App extends React.Component { onChange={this.onImageAssessmentChange} /> + +
+ Style Transfer model: + + + + + +
)} diff --git a/frontend/static/frontend/js/app.client.js b/frontend/static/frontend/js/app.client.js index ddd1f75..3be7132 100644 --- a/frontend/static/frontend/js/app.client.js +++ b/frontend/static/frontend/js/app.client.js @@ -1,9 +1,9 @@ -!function(e){var t={};function n(r){if(t[r])return t[r].exports;var o=t[r]={i:r,l:!1,exports:{}};return e[r].call(o.exports,o,o.exports,n),o.l=!0,o.exports}n.m=e,n.c=t,n.d=function(e,t,r){n.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:r})},n.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},n.t=function(e,t){if(1&t&&(e=n(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var o in e)n.d(r,o,function(t){return e[t]}.bind(null,o));return r},n.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return n.d(t,"a",t),t},n.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},n.p="",n(n.s=267)}([function(e,t,n){"use strict";e.exports=n(100)},function(e,t,n){e.exports=n(107)()},function(e,t,n){"use strict";n.r(t),function(e){n.d(t,"flush",function(){return i}),n.d(t,"hydrate",function(){return u}),n.d(t,"cx",function(){return l}),n.d(t,"merge",function(){return s}),n.d(t,"getRegisteredStyles",function(){return c}),n.d(t,"injectGlobal",function(){return f}),n.d(t,"keyframes",function(){return p}),n.d(t,"css",function(){return d}),n.d(t,"sheet",function(){return h}),n.d(t,"caches",function(){return m});var r=n(99),o=void 0!==e?e:{},a=Object(r.a)(o),i=a.flush,u=a.hydrate,l=a.cx,s=a.merge,c=a.getRegisteredStyles,f=a.injectGlobal,p=a.keyframes,d=a.css,h=a.sheet,m=a.caches}.call(this,n(41))},function(e,t,n){"use strict";t.__esModule=!0;var r=function(e){return e&&e.__esModule?e:{default:e}}(n(87));t.default=function(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!==(void 0===t?"undefined":(0,r.default)(t))&&"function"!=typeof t?e:t}},function(e,t,n){"use strict";n.r(t);var r=n(0),o=n.n(r),a=n(8),i=n.n(a),u=n(13),l=n.n(u),s=n(5),c=n.n(s),f=n(3),p=n.n(f),d=n(6),h=n.n(d);function m(){var e=this.constructor.getDerivedStateFromProps(this.props,this.state);null!==e&&void 0!==e&&this.setState(e)}function y(e){this.setState(function(t){var n=this.constructor.getDerivedStateFromProps(e,t);return null!==n&&void 0!==n?n:null}.bind(this))}function v(e,t){try{var n=this.props,r=this.state;this.props=e,this.state=t,this.__reactInternalSnapshotFlag=!0,this.__reactInternalSnapshot=this.getSnapshotBeforeUpdate(n,r)}finally{this.props=n,this.state=r}}function g(e){var t=e.prototype;if(!t||!t.isReactComponent)throw new Error("Can only polyfill class components");if("function"!=typeof e.getDerivedStateFromProps&&"function"!=typeof t.getSnapshotBeforeUpdate)return e;var n=null,r=null,o=null;if("function"==typeof t.componentWillMount?n="componentWillMount":"function"==typeof t.UNSAFE_componentWillMount&&(n="UNSAFE_componentWillMount"),"function"==typeof t.componentWillReceiveProps?r="componentWillReceiveProps":"function"==typeof t.UNSAFE_componentWillReceiveProps&&(r="UNSAFE_componentWillReceiveProps"),"function"==typeof t.componentWillUpdate?o="componentWillUpdate":"function"==typeof t.UNSAFE_componentWillUpdate&&(o="UNSAFE_componentWillUpdate"),null!==n||null!==r||null!==o){var a=e.displayName||e.name,i="function"==typeof e.getDerivedStateFromProps?"getDerivedStateFromProps()":"getSnapshotBeforeUpdate()";throw Error("Unsafe legacy lifecycles will not be called for components using new component APIs.\n\n"+a+" uses "+i+" but also contains the following legacy lifecycles:"+(null!==n?"\n "+n:"")+(null!==r?"\n "+r:"")+(null!==o?"\n "+o:"")+"\n\nThe above lifecycles should be removed. Learn more about this warning here:\nhttps://fb.me/react-async-component-lifecycle-hooks")}if("function"==typeof e.getDerivedStateFromProps&&(t.componentWillMount=m,t.componentWillReceiveProps=y),"function"==typeof t.getSnapshotBeforeUpdate){if("function"!=typeof t.componentDidUpdate)throw new Error("Cannot polyfill getSnapshotBeforeUpdate() for components that do not define componentDidUpdate() on the prototype");t.componentWillUpdate=v;var u=t.componentDidUpdate;t.componentDidUpdate=function(e,t,n){var r=this.__reactInternalSnapshotFlag?this.__reactInternalSnapshot:n;u.call(this,e,t,r)}}return e}m.__suppressDeprecationWarning=!0,y.__suppressDeprecationWarning=!0,v.__suppressDeprecationWarning=!0;var b=n(58),w=n.n(b),x=n(37),_=n.n(x),k=n(97),E=n.n(k),S=n(59),P=n(38);n.d(t,"mapProps",function(){return j}),n.d(t,"withProps",function(){return A}),n.d(t,"withPropsOnChange",function(){return z}),n.d(t,"withHandlers",function(){return D}),n.d(t,"defaultProps",function(){return M}),n.d(t,"renameProp",function(){return I}),n.d(t,"renameProps",function(){return V}),n.d(t,"flattenProp",function(){return B}),n.d(t,"withState",function(){return W}),n.d(t,"withStateHandlers",function(){return H}),n.d(t,"withReducer",function(){return q}),n.d(t,"branch",function(){return K}),n.d(t,"renderComponent",function(){return G}),n.d(t,"renderNothing",function(){return Q}),n.d(t,"shouldUpdate",function(){return Z}),n.d(t,"pure",function(){return J}),n.d(t,"onlyUpdateForKeys",function(){return ee}),n.d(t,"onlyUpdateForPropTypes",function(){return te}),n.d(t,"withContext",function(){return ne}),n.d(t,"getContext",function(){return re}),n.d(t,"lifecycle",function(){return oe}),n.d(t,"toClass",function(){return ie}),n.d(t,"withRenderProps",function(){return ue}),n.d(t,"setStatic",function(){return C}),n.d(t,"setPropTypes",function(){return le}),n.d(t,"setDisplayName",function(){return O}),n.d(t,"compose",function(){return se}),n.d(t,"getDisplayName",function(){return T}),n.d(t,"wrapDisplayName",function(){return N}),n.d(t,"isClassComponent",function(){return ae}),n.d(t,"createSink",function(){return ce}),n.d(t,"componentFromProp",function(){return fe}),n.d(t,"nest",function(){return pe}),n.d(t,"hoistStatics",function(){return de}),n.d(t,"componentFromStream",function(){return ge}),n.d(t,"componentFromStreamWithConfig",function(){return ve}),n.d(t,"mapPropsStream",function(){return xe}),n.d(t,"mapPropsStreamWithConfig",function(){return we}),n.d(t,"createEventHandler",function(){return ke}),n.d(t,"createEventHandlerWithConfig",function(){return _e}),n.d(t,"setObservableConfig",function(){return me}),n.d(t,"shallowEqual",function(){return l.a});var C=function(e,t){return function(n){return n[e]=t,n}},O=function(e){return C("displayName",e)},T=function(e){return"string"==typeof e?e:e?e.displayName||e.name||"Component":void 0},N=function(e,t){return t+"("+T(e)+")"},j=function(e){return function(t){var n=Object(r.createFactory)(t);return function(t){return n(e(t))}}},A=function(e){return j(function(t){return i()({},t,"function"==typeof e?e(t):e)})},U=function(e,t){for(var n={},r=0;r1?r-1:0),a=1;a1&&void 0!==arguments[1]?arguments[1]:$;return t.setState(function(t){var r=t.stateValue;return{stateValue:n(r,e)}},function(){return r(t.state.stateValue)})},o=e,p()(t,o)}return h()(a,r),a.prototype.initializeStateValue=function(){return void 0!==o?"function"==typeof o?o(this.props):o:n(void 0,{type:"@@recompose/INIT"})},a.prototype.render=function(){var n;return u(i()({},this.props,((n={})[e]=this.state.stateValue,n[t]=this.dispatch,n)))},a}(r.Component)}},Y=function(e){return e},K=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:Y;return function(o){var a=void 0,i=void 0;return function(u){return e(u)?(a=a||Object(r.createFactory)(t(o)))(u):(i=i||Object(r.createFactory)(n(o)))(u)}}},G=function(e){return function(t){var n=Object(r.createFactory)(e);return function(e){return n(e)}}},X=function(e){function t(){return c()(this,t),p()(this,e.apply(this,arguments))}return h()(t,e),t.prototype.render=function(){return null},t}(r.Component),Q=function(e){return X},Z=function(e){return function(t){var n=Object(r.createFactory)(t);return function(t){function r(){return c()(this,r),p()(this,t.apply(this,arguments))}return h()(r,t),r.prototype.shouldComponentUpdate=function(t){return e(this.props,t)},r.prototype.render=function(){return n(this.props)},r}(r.Component)}},J=function(e){return Z(function(e,t){return!l()(e,t)})(e)},ee=function(e){return Z(function(t,n){return!l()(U(n,e),U(t,e))})},te=function(e){var t=e.propTypes,n=_()(t||{});return ee(n)(e)},ne=function(e,t){return function(n){var o=Object(r.createFactory)(n),a=function(e){function n(){var r,o,a;c()(this,n);for(var i=arguments.length,u=Array(i),l=0;l=200&&e<300},headers:{common:{Accept:"application/json, text/plain, */*"}}};r.forEach(["delete","get","head"],function(e){u.headers[e]={}}),r.forEach(["post","put","patch"],function(e){u.headers[e]=r.merge(a)}),e.exports=u}).call(this,n(75))},function(e,t,n){var r=n(18);e.exports=function(e,t){if(!r(e))return e;var n,o;if(t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;if("function"==typeof(n=e.valueOf)&&!r(o=n.call(e)))return o;if(!t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;throw TypeError("Can't convert object to primitive value")}},function(e,t){e.exports=function(e){if(void 0==e)throw TypeError("Can't call method on "+e);return e}},function(e,t){var n=Math.ceil,r=Math.floor;e.exports=function(e){return isNaN(e=+e)?0:(e>0?r:n)(e)}},function(e,t,n){var r=n(48)("keys"),o=n(34);e.exports=function(e){return r[e]||(r[e]=o(e))}},function(e,t,n){var r=n(10),o=n(11),a=o["__core-js_shared__"]||(o["__core-js_shared__"]={});(e.exports=function(e,t){return a[e]||(a[e]=void 0!==t?t:{})})("versions",[]).push({version:r.version,mode:n(33)?"pure":"global",copyright:"© 2018 Denis Pushkarev (zloirock.ru)"})},function(e,t){e.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(e,t){t.f=Object.getOwnPropertySymbols},function(e,t,n){var r=n(45);e.exports=function(e){return Object(r(e))}},function(e,t){e.exports={}},function(e,t,n){var r=n(24),o=n(221),a=n(49),i=n(47)("IE_PROTO"),u=function(){},l=function(){var e,t=n(83)("iframe"),r=a.length;for(t.style.display="none",n(222).appendChild(t),t.src="javascript:",(e=t.contentWindow.document).open(),e.write("