라즈베리파이-앱 연결

라즈베리파이4b와 Flutter 앱이 데이터를 주고받기 위해 서버 이용을 시도하였습니다.

socket, flask, firebase를 이용하는 세가지 방법을 시도하였습니다.

언어는 python을 사용하였습니다.

1) socket 이용

https://docs.python.org/ko/3/library/socket.html

socket — Low-level networking interface

다음 Python Documentation socket부분의 예시를 참고하였습니다.

# Echo server program
import socket

HOST = ''                 # Symbolic name meaning all available interfaces
PORT = 50007              # Arbitrary non-privileged port
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    s.bind((HOST, PORT))
    s.listen(1)
    conn, addr = s.accept()
    with conn:
        print('Connected by', addr)
        while True:
            data = conn.recv(1024)
            if not data: break
            conn.sendall(data)

 

이를 참고하여 기존 Python 코드에 socket 코드를 추가하였습니다.

-> socket_vr.py

import cv2
import numpy as np
import datetime
import sys, os
import requests
import firebase_admin
import torch
import json
import yolov5.utils.torch_utils
import socket

from firebase_admin import credentials
from firebase_admin import storage
from firebase_admin import firestore
from uuid import uuid4
from flask import Flask,request

app = Flask(__name__)
    
#load model
model_name='/home/ksb/yolov5/coopgo/coopgo_model/weights/best.pt'
model = torch.hub.load(os.getcwd(), 'custom', source='local', path = model_name, force_reload = True)

#firebase initialization
PROJECT_ID = "coopgo-b58bd"
cred = credentials.Certificate("/home/ksb/coopgo-b58bd-firebase-adminsdk-q4m0h-913cfe20db.json")
default_app = firebase_admin.initialize_app(cred,{'storageBucket':f"{PROJECT_ID}.appspot.com"})
db = firestore.client()
bucket = storage.bucket()
print("firebase setup end")

#capture
threshold_move = 50
diff_compare = 10 
cap = cv2.VideoCapture(-1)
cap.set(cv2.CAP_PROP_FRAME_WIDTH,320)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT,240)
picture_directory = "/home/ksb/webcam/" 

print("capture setup end")

def savePhoto(frame,filename,img_path):
    img_path = picture_directory+filename
    cv2.imwrite(img_path, frame)

def runModel(img_path, objects) : 
    objects = []
    print("Run Model")
    results = model(img_path)
    detections = results.pandas().xyxy[0]

    for _, detection in detections.iterrows():
        cls_name = detection['name']
        if cls_name not in objects:
            objects[cls_name] = 0
        objects[cls_name]+= 1
        
    if objects :                          #if any object is detected
        results.show()
        #db.collection(u'detectionResult').document(u'5znMimhJJau6OYtILKM4').set(results)
        #results.save(labels=True,save_dir= img_path)
        #uploadPhoto(results)
        #db.collection(u'detectionResult').document(str_now).set(objects)
        print("DB upload")

#upload photo on firebase storage        
def uploadPhoto(file):
    blob = bucket.blob('pictures/' + file)
    new_token = uuid4()
    metadata = {"firebaseStorageDownloadTokens": new_token}  # access token 필요
    blob.metadata = metadata
    blob.upload_from_filename(filename=picture_directory+file)
    print("Photo delete on local space & Uploade Complete")
    print(blob.public_url)

#do capture 
while True: 
    print("listening...")
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:    #socket
        s.bind((HOST,PORT))
        s.listen(1)
        conn,addr = s.accept()
        with conn:
            print(f'Connected by {addr}')
            while True:
                data = conn.recv(1024)
                if data:
                    print(data.decode())
                    key = data['id']
                   
                    response_data = "Received POST request"   # Response
                    response = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\n\r\n{}".format(len(response_data), response_data)
                    conn.sendall(response.encode())
                
                    capture_cnt = 5       # number of times capturing objs
                    while capture_cnt :
                        capture_cnt-= 1 
                       
                        now = datetime.datetime.now()
                        str_now = str(now.strftime('%y%m%d%H%M%S'))
                        filename = str_now + '.jpg'   # name filename
                        
                        ret, img= cap.read()      #read img
                        img_path = ""
                        
                        #save & upload photo
                        savePhoto(img, filename, img_path)
                        uploadPhoto(filename)
                        
                        #find one optimal detection out of 5 times of capturing
                        objects = {}
                        max_obj_num = 0
                        max_objects = {}
                        runModel(img_path, objects)     # run yolov5 coop-go model
                        
                        if(objects.size() > max_obj_num):    #update max_obj_num & max_objects
                            max_obj_num = objects.size()
                            max_objects = objects
                            
                        #send data 
                        conn.sendall(max_objects)
                        #upload collection on firestore
                        db.collection(u'detectionResult').document(key).set(max_objects)

2)  Python Flask 웹 프레임워크 사용

플라스크 웹 서버 이용하기

 

host는 raspberrypi, port=8000을 사용하였고,

전송된 데이터를 json형식으로 바꾸고 parsed 변수에 넣었습니다.

parsed가 None이 아닐 때, capture()를 시행합니다.

그리고 클라이언트에게 data가 전송됐음을 알리기 위해 response_data를 return 합니다.

#flask
@app.route('/api/qr-detection',methods=['POST'])
def handle_post_request():
    if request.method=='POST':
        parsed = json.loads(request.data)
        print(parsed)
        if parsed :
            capture("---") # for test
        response_data="Received Post request"
        return response_data,200

if __name__=='__main__':
    app.run(host='raspberrypi',port=8000)

 

기존 Python 코드에 flask 코드를 추가하였습니다.

 

import cv2
import numpy as np
import datetime
import sys, os
import requests
import firebase_admin
import torch
import json
import yolov5.utils.torch_utils
import socket
import json
from firebase_admin import credentials
from firebase_admin import storage
from firebase_admin import firestore
from uuid import uuid4
from flask import Flask,request
app = Flask(__name__)
#load model
model_name='/home/ksb/yolov5/coopgo/coopgo_model/weights/best.pt'
model = torch.hub.load(os.getcwd(), 'custom', source='local', path = model_name, force_reload = True)
#firebase initialization
PROJECT_ID = "coopgo-b58bd"
cred = credentials.Certificate("/home/ksb/coopgo-b58bd-firebase-adminsdk-q4m0h-913cfe20db.json")
default_app = firebase_admin.initialize_app(cred,{'storageBucket':f"{PROJECT_ID}.appspot.com"})
db = firestore.client()
bucket = storage.bucket()#기본 버킷 사용
print("firebase setup end")

#capture
threshold_move = 50
diff_compare = 10 

try : 
    cap = cv2.VideoCapture(-1)
except:
    print("No Camera")
    quit()

cap.set(cv2.CAP_PROP_FRAME_WIDTH,320)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT,240)
picture_directory = "/home/ksb/webcam/" 
print("capture setup end")

def savePhoto(frame,filename):
    img_path = picture_directory+filename
    cv2.imwrite(img_path, frame)
    return img_path

def runModel(objects, img_path) : 
    print("Run Model")  
    objects = {}
    results = model(img_path)
    detections = results.pandas().xyxy[0]
    for _, detection in detections.iterrows():
        cls_name = detection['name']
        if cls_name not in objects:
            objects[cls_name] = 0
        objects[cls_name]+= 1
    print(objects)    
    if(objects != {}) :
        results.show()
        results.save(labels=True,save_dir= img_path+'res')
        uploadPhoto(results)
    return objects    

def uploadPhoto(file):
    blob = bucket.blob('pictures/' + file)
    new_token = uuid4()
    metadata = {"firebaseStorageDownloadTokens": new_token}  
    blob.metadata = metadata
    blob.upload_from_filename(filename=picture_directory+file)
    print("사진 delete & 업로드 완료")
    print(blob.public_url)


def capture(key):
    print(key)
    objects = {}
    max_obj_num = 0
    max_objects = {}
    capture_cnt = 5 # take 5 pictures in a row
    while capture_cnt > 0: 
        capture_cnt -= 1 
        now = datetime.datetime.now()
        str_now = str(now.strftime('%y%m%d%H%M%S'))
        filename = str_now + '.jpg' 
        ret, img= cap.read() # take a picture
        img_path = savePhoto(img, filename)
        print(img_path)
        print("image saved")
        #count num of objects
        objects = runModel(objects, img_path) # model
        if(len(objects) > max_obj_num):
            max_obj_num = objects.size()
            max_objects = objects
    db.collection(u'detection').document(key).set(max_objects)
    print("firestore upload")
    uploadPhoto(filename)        
    print("storage upload")        

                       
#flask
@app.route('/api/qr-detection',methods=['POST'])
def handle_post_request():
    if request.method=='POST':
        parsed = json.loads(request.data)
        print(parsed)
        if parsed :
            capture("2076196") #test
        response_data="Received Post request"
        return response_data,200

if __name__=='__main__':
    app.run(host='raspberrypi',port=8000)

 

3) firebase 사용하기

Google firebase에서 제공하는 서버를 사용하기

https://firebase.google.com/docs/firestore/query-data/listen#python

Cloud Firestore로 실시간 업데이트 가져오기  |  Firebase

위의 문서에 있는 예시를 참고하였습니다. 예시는 다음과 같습니다.

# Create an Event for notifying main thread.
callback_done = threading.Event()

# Create a callback on_snapshot function to capture changes
def on_snapshot(doc_snapshot, changes, read_time):
    for doc in doc_snapshot:
        print(f'Received document snapshot: {doc.id}')
    callback_done.set()

doc_ref = db.collection('cities').document('SF')

# Watch the document
doc_watch = doc_ref.on_snapshot(on_snapshot)snippets.py

위의 코드를 다음과 같이 변형하였습니다.

추가된 document가 오직 1개일 때, 추가된 document의 id를 출력하고 capture()를 수행합니다.

같은 id로 여러번 capture()가 수행되는 것을 막기 위해 cur_document 와 pre_document가 다를 때만 capture()가 수행되도록 하였습니다.

def on_snapshot(col_snapshot, changes, read_time):
    global pre_document
    print('ready')
    if len(changes) == 1:
        for change in changes:
            if change.type.name == 'ADDED':
                print(f'{change.document.id}')
                cur_document = change.document.id
                if pre_document == cur_document:
                    break
                pre_document = cur_document
                doc_ref = db.collection(u'qr').document(cur_document)
                #capture(doc_ref.id)
                capture(cur_document)
    

# Watch the document
query_watch = col_query.on_snapshot(on_snapshot)
while True:
    callback_done.wait() 
    callback_done.clear()

다음은 전체 코드입니다.

import cv2
import numpy as np
import datetime
import sys, os
import requests
import firebase_admin
import torch
import json
import yolov5.utils.torch_utils
import socket
import json, threading
from PIL import Image
from firebase_admin import credentials
from firebase_admin import storage
from firebase_admin import firestore
from uuid import uuid4
from flask import Flask,request

app = Flask(__name__)

#load model
#model =torch.load('/home/ksb/yolov5/coopgo/coopgo_model/weights/best.pt')
model_name = '/home/ksb/yolov5/coopgo/coopgo_model/weights/best.pt'
model = torch.hub.load(os.getcwd(), 'custom', source='local', path = model_name, force_reload = True)

#firebase initialization
PROJECT_ID = "coopgo-b58bd"
cred = credentials.Certificate("/home/ksb/coopgo-b58bd-firebase-adminsdk-q4m0h-913cfe20db.json")
default_app = firebase_admin.initialize_app(cred,{'storageBucket':f"{PROJECT_ID}.appspot.com"})
db = firestore.client()
bucket = storage.bucket()#기본 버킷 사용
print("firebase setup end")

#capture
threshold_move = 50
diff_compare = 10 

#listening firestore 
pre_document = "hello"
callback_done=threading.Event()
col_query = db.collection(u'qr')

try : 
    cap = cv2.VideoCapture(-1)
except:
    print("No Camera")
    quit()
    
cap.set(cv2.CAP_PROP_FRAME_WIDTH,320)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT,240)
picture_directory = "/home/ksb/webcam/" 

print("camera setup end")

def savePhoto(frame,filename):
    img_path = picture_directory+filename
    cv2.imwrite(img_path, frame)
    return img_path

def runModel(img_path) :
    objects = {} 
    #img = Image.open(img_path)
    print("Run Model")  
    results = model(img_path)
    detections = results.pandas().xyxy[0]
            
    for _, detection in detections.iterrows():
        cls_name = detection['name']
        if cls_name not in objects:
            objects[cls_name] = 0
        objects[cls_name]+= 1
    #print the objects    
    print(objects)    
    if(objects) :
        results.show()
        
    return objects    

def uploadPhoto(filename):
    blob = bucket.blob('pictures/' + filename)
    new_token = uuid4()
    metadata = {"firebaseStorageDownloadTokens": new_token}  
    blob.metadata = metadata
    blob.upload_from_filename(filename=picture_directory+filename)
    print("사진 delete & 업로드 완료")
    print(blob.public_url)

def capture(key):
    print(key)
    objects = {}
    max_obj_num = 0
    max_objects = {}
    capture_cnt = 5 # take 5 pictures in a row
    
    while capture_cnt > 0: 
        capture_cnt -= 1 
        now = datetime.datetime.now()
        str_now = str(now.strftime('%y%m%d%H%M%S'))
        filename = str_now + '.jpg' 
        ret, img= cap.read() # take a picture
        img_path = savePhoto(img, filename)
        print(img_path)
        print("image saved")
        #count num of objects
        objects = runModel(img_path) # model
        if(len(objects) > max_obj_num):
            max_obj_num = len(objects)
            max_objects = objects
            print(f'max objects : {0}'.format(objects))
            
    db.collection(u'detection').document(key).set(max_objects)
    print("firestore upload")
    uploadPhoto(filename)        
    print("storage upload")   
    
                        
# Create a callback on_snapshot function to capture changes

def on_snapshot(col_snapshot, changes, read_time):
    global pre_document
    print('ready')
    if len(changes) == 1:
        for change in changes:
            if change.type.name == 'ADDED':
                print(f'{change.document.id}')
                cur_document = change.document.id
                if pre_document == cur_document:
                    break
                pre_document = cur_document
                doc_ref = db.collection(u'qr').document(cur_document)
                #capture(doc_ref.id)
                capture(cur_document)
    

# Watch the document
query_watch = col_query.on_snapshot(on_snapshot)
while True:
    callback_done.wait() 
    callback_done.clear()