Popularity (#10)

* Add popularity

* Update popularity model to dockerfile

* Fix dockerfile

* Revert "Fix dockerfile"

This reverts commit ea24fba604.

* Fix dockerfile

* Fix filename

* Fix filename

* Fix mkdir

* Fix mkdir

* Fix #9

* Zip popularity model

* Fix model filename

* Remove comixify volume

* Fix minimum number of segments

* Add debug print

* Update model 1 file

docker-compose-prod.yml

@@ -14,7 +14,6 @@ services:
     build: .
     runtime: nvidia
     volumes:
-      - .:/comixify
       - static_volume:/comixify/static
       - media_volume:/comixify/media
     networks:

docker-compose.yml

@@ -14,7 +14,6 @@ services:
     build: .
     runtime: nvidia
     volumes:
-      - .:/comixify
       - static_volume:/comixify/static
       - media_volume:/comixify/media
     networks:

dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y apt-utils software-properties-common &&
     liblmdb-dev libopencv-dev libprotobuf-dev \
     libsnappy-dev protobuf-compiler \
     python-numpy python-setuptools python-scipy \
-    libavformat-dev libswscale-dev && \
+    libavformat-dev libswscale-dev unzip && \
     python3.6 -m pip install --upgrade pip && \
     python3.6 -m pip install jupyter ipywidgets jupyterlab && \
     python3.6 -m pip install tensorflow-gpu h5py keras && \
@@ -44,7 +44,9 @@ RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig &&
 
 WORKDIR /comixify
 COPY . /comixify
-RUN python3.6 -m pip install -r requirements.txt
+RUN unzip popularity/pretrained_model/svr_test_11.10.sk.zip -d popularity/pretrained_model/ && \
+    python3.6 -m pip install -r requirements.txt
+
 
 # Port to expose
 EXPOSE 8008

frontend/client/App.js

@@ -95,7 +95,7 @@ class App extends React.Component {
 			console.error(rejected);
 			this.setState({
 				drop_errors: ["Maximum size for single video is 50MB"],
-				stata: App.appStates.DROP_ERROR
+				state: App.appStates.DROP_ERROR
 			});
 			return;
 		}

keyframes/keyframes.py

@@ -17,6 +17,7 @@ import logging
 
 from utils import jj
 from keyframes_rl.models import DSN
+from popularity.models import PopularityPredictor
 from keyframes.kts import cpd_auto
 from keyframes.utils import batch
 
@@ -30,9 +31,11 @@ class KeyFramesExtractor:
         frames_paths, all_frames_tmp_dir = cls._get_all_frames(video, mode=frames_mode)
         frames = cls._get_frames(frames_paths)
         features = cls._get_features(frames, gpu, features_batch_size)
-        change_points, frames_per_segment = cls._get_segments(features)
+        norm_features = normalize(features)
-        probs = cls._get_probs(features, gpu, mode=rl_mode)
+        change_points, frames_per_segment = cls._get_segments(norm_features)
-        chosen_frames = cls._get_chosen_frames(frames, probs, change_points, frames_per_segment)
+        probs = cls._get_probs(norm_features, gpu, mode=rl_mode)
+        keyframes = cls._get_keyframes(frames, probs, change_points, frames_per_segment)
+        chosen_frames = cls._get_popularity_chosen_frames(keyframes, features)
         return chosen_frames
 
     @staticmethod
@@ -97,7 +100,6 @@ class KeyFramesExtractor:
             temp = net.blobs["pool5/7x7_s1"].data[0:n_batch]
             temp = temp.squeeze().copy()
             features[idx_batch * batch_size:idx_batch * batch_size + n_batch] = temp
-        normalize(features, copy=False)
         return features.astype(np.float32)
 
     @staticmethod
@@ -128,7 +130,7 @@ class KeyFramesExtractor:
         return probs
 
     @staticmethod
-    def _get_chosen_frames(frames, probs, change_points, frames_per_segment, min_keyframes=10):
+    def _get_keyframes(frames, probs, change_points, frames_per_segment, min_keyframes=20):
         gts = []
         s = 0
         for q in frames_per_segment:
@@ -146,20 +148,37 @@ class KeyFramesExtractor:
                 x = low + np.argmax(probs[low:high])
             chosen_frames.append({
                 "index": x,
-                "frame": frames[x]
+                "frame": img_as_ubyte(frames[x])[..., ::-1]
             })
         chosen_frames.sort(key=lambda k: k['index'])
-        chosen_frames = [img_as_ubyte(o["frame"])[..., ::-1] for o in chosen_frames]
         return chosen_frames
 
+    @staticmethod
+    def _get_popularity_chosen_frames(frames, features, n_frames=10):
+        model_cache_key = "popularity_model_cache"
+        model = cache.get(model_cache_key)  # get model from cache
+
+        if model is None:
+            model = PopularityPredictor()
+            cache.set(model_cache_key, model, None)
+
+        for frame in frames:
+            x = features[frame["index"]]
+            frame["popularity"] = model.get_popularity_score(x).squeeze()
+
+        chosen_frames = sorted(frames, key=lambda k: k['popularity'], reverse=True)
+        chosen_frames = chosen_frames[0:n_frames]
+        chosen_frames.sort(key=lambda k: k['index'])
+        return [o["frame"] for o in chosen_frames]
+
     @staticmethod
     def _get_segments(features):
         K = np.dot(features, features.T)
         n_frames = int(K.shape[0])
-        min_segments = int(ceil(n_frames / 10))
+        min_segments = int(ceil(n_frames / 20))
-        min_segments = max(10, min_segments)
+        min_segments = max(20, min_segments)
         min_segments = min(n_frames - 1, min_segments)
-        cps, scores = cpd_auto(K, min_segments, 1)
+        cps, scores = cpd_auto(K, min_segments, 1, min_segments=min_segments)
         change_points = [
             [0, cps[0] - 1]
         ]
@@ -169,4 +188,5 @@ class KeyFramesExtractor:
             frames_per_segment.append(int(cps[j + 1] - cps[j]))
         frames_per_segment.append(int(len(features) - cps[len(cps) - 1]))
         change_points.append([cps[len(cps) - 1], len(features) - 1])
+        print("Number of segments: " + str(len(frames_per_segment)))
         return change_points, frames_per_segment

keyframes/kts/__init__.py

@@ -85,7 +85,7 @@ def cpd_nonlin(K, ncp, lmin=1, lmax=100000, backtrack=True, verbose=True, out_sc
     return cps, scores
 
 
-def cpd_auto(K, ncp, vmax, desc_rate=15, min_segments=10, **kwargs):
+def cpd_auto(K, ncp, vmax, desc_rate=15, min_segments=20, **kwargs):
     """Main interface
     Detect change points automatically selecting their number
         K       - kernel between each pair of frames in video

popularity/models.py

@@ -0,0 +1,16 @@
+import pickle
+import os.path
+
+MODEL_PATH = 'popularity/pretrained_model/svr_test_11.10.sk'
+
+
+class PopularityPredictor:
+    def __init__(self):
+        if not os.path.exists(MODEL_PATH):
+            print("Model file does not exist.")
+        with open(MODEL_PATH, 'rb') as fp:
+            self.svr = pickle.load(fp, encoding='latin1')
+
+    def get_popularity_score(self, image_feature):
+        image_feature = image_feature.reshape(1, -1)
+        return self.svr.predict(image_feature)