As a part of my project involving conversion of sign language to english , I developed .tflite model and did predictions successfully using opencv.
Now i want to implement web application for this . On the web page , there will be live streaming going on and frames will be predicted and the corresponding predicted text has to be displayed on the top of page
This camera.py will take image from live streaming and predict it and return the modified image along with the predicted sequence to app.py.
Now the problem is how do i continously return these both(frame and the sequence) to index.html
What I have tried:
My app.py code is as follows
from flask import Flask, render_template, Response
from camera import Video
import time
sequence = ''
app=Flask(__name__)
print("before gen",time.perf_counter())
def gen(camera):
while True:
frame,sequence=camera.get_frame()
print(sequence)
sequence = sequence.upper()
yield(b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
@app.route('/video')
def video():
return Response(gen(Video()),mimetype='multipart/x-mixed-replace; boundary=frame')
@app.route('/')
def index():
return render_template('index.html' , sequence = sequence)
app.run(debug=True)
The camera.py code is as follows
import os
import time
import numpy as np
import tensorflow.lite as tflite
import cv2
# Disable tensorflow compilation warnings
os.environ['TF_CPP_MIN_LOG_LEVEL']='3'
import tensorflow as tf
class Video(object):
def __init__(self):
self.video=cv2.VideoCapture(0)
self.model()
def __del__(self):
self.video.release()
def model(self):
with open("logs/trained_labels.txt", 'r') as f:
self.labels = [line.strip() for line in f.readlines()]
self.interpreter = tflite.Interpreter(model_path="model_unquant.tflite")
self.interpreter.allocate_tensors()
self.input_details = self.interpreter.get_input_details()
self.input_index = self.input_details[0]['index']
self.c= 0
self.res, self.score= '', 0.0
self.i = 0
self.mem = ''
self.consecutive = 0
self.sequence = ''
def predict(self,image_data):
#print(image_data.shape)
input_data = np.expand_dims(image_data, axis=0).astype(np.float32) # expand to 4-dim
#print(input_data)
self.interpreter.set_tensor(self.input_index,input_data)
self.interpreter.invoke()
output_details = self.interpreter.get_output_details()
output_data = self.interpreter.get_tensor(output_details[0]['index'])
output_data = np.squeeze(output_data)
top_k = output_data.argsort()[-len(output_data):][::-1]
a = []
for i in top_k:
sign = self.labels[i]
score = output_data[i]
sign = self.labels[i]
score = output_data[i]
a.append((sign,score))
a = sorted(a, key = lambda x: x[1],reverse=True)
return a[0][0], a[0][1]
def get_frame(self):
print("getframe",time.perf_counter())
ret,frame=self.video.read()
img = cv2.flip(frame, 1)
if ret:
x1, y1, x2, y2 = 200, 100, 424, 324
img_cropped = img[y1:y2, x1:x2]
self.c+= 1
#image_data = cv2.imencode('.jpg', img_cropped)[1].tostring()
if self.i == 4:
res_tmp, self.score= self.predict(img_cropped)
self.res= res_tmp
self.i = 0
if self.mem == self.res:
self.consecutive += 1
else:
self.consecutive = 0
if self.consecutive == 2 and self.res not in ['nothing']:
if self.res== 'space':
self.sequence += ' '
elif self.res== 'del':
self.sequence = self.sequence[:-1]
else:
self.sequence += self.res
self.consecutive = 0
print("after getframe",time.perf_counter())
self.i += 1
cv2.putText(img, '%s' % (self.res.upper()), (250,400), cv2.FONT_HERSHEY_SIMPLEX, 3, (255,255,255), 4)
cv2.putText(img, '%.3f' % (float(self.score)), (200,450), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255))
self.mem = self.res
cv2.rectangle(img, (x1, y1), (x2, y2), (18,31,150), 3)
ret,jpg=cv2.imencode('.jpg',img)
return jpg.tobytes(),self.sequence.upper()
the model and labels files are in the following link
https://www.dropbox.com/sh/nej0cvmo2w34g1a/AAAYeEd4A6f4VzcdyTVP30Hia?dl=0
These 2 fields (image and sequence )should be keep on updating as long as the application is running. How do i send these both at a time to html page. how do i use both response and render_template keywords at a time. Please help me..