<jupyter_start><jupyter_text>0. Install and Import Dependencies<jupyter_code>!pip install mediapipe opencv-python pandas scikit-learn
import mediapipe as mp # Import mediapipe
import cv2 # Import opencv
import csv
mp_drawing = mp.solutions.drawing_utils # Drawing helpers
mp_holistic = mp.solutions.holistic # Mediapipe Solutions<jupyter_output><empty_output><jupyter_text>1. Make Some Detections<jupyter_code>with open('coords.csv', mode='w', newline='') as f:
    csv_writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    csv_writer.writerow(landmarks)<jupyter_output><empty_output><jupyter_text>Webcam version<jupyter_code>cap = cv2.VideoCapture(0)
# Initiate holistic model
with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
    
    while cap.isOpened():
        ret, frame = cap.read()
        
        # Recolor Feed
        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        image.flags.writeable = False        
        
        # Make Detections
        results = holistic.process(image)
        # print(results.face_landmarks)
        
        # face_landmarks, pose_landmarks, left_hand_landmarks, right_hand_landmarks
        
        # Recolor image back to BGR for rendering
        image.flags.writeable = True   
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        
        # 1. Draw face landmarks
        mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(80,110,10), thickness=1, circle_radius=1),
                                 mp_drawing.DrawingSpec(color=(80,256,121), thickness=1, circle_radius=1)
                                 )
        
        # 2. Right hand
        mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(80,22,10), thickness=2, circle_radius=4),
                                 mp_drawing.DrawingSpec(color=(80,44,121), thickness=2, circle_radius=2)
                                 )

        # 3. Left Hand
        mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4),
                                 mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2)
                                 )

        # 4. Pose Detections
        mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4),
                                 mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2)
                                 )
                        
        cv2.imshow('Raw Webcam Feed', image)

        if cv2.waitKey(10) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()<jupyter_output><empty_output><jupyter_text>Youtube Video Version<jupyter_code>vids = [{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "70311", "cend": "72063"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "539016", "cend": "539746"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "558453", "cend": "558966"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "578393", "cend": "579337"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "582497", "cend": "583735"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "584176", "cend": "584677"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "585752", "cend": "586112"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "595601", "cend": "596045"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "597100", "cend": "597559"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "602642", "cend": "602957"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "606490", "cend": "607434"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "618813", "cend": "619334"},
{"cID": "EdVueeaNAK0", "title": "Ollie", "cstart": "630671", "cend": "632162"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "16015", "cend": "21669"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "28784", "cend": "33723"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "100735", "cend": "105913"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "121867", "cend": "126552"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "159667", "cend": "161173"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "161497", "cend": "165076"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "165590", "cend": "169727"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "178468", "cend": "180045"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "182830", "cend": "188237"},
{"cID": "KJnZvKwgZaA", "title": "Ollie", "cstart": "195547", "cend": "199861"},
{"cID": "fp_UMkZabvk", "title": "Nollie", "cstart": "6442", "cend": "13078"},
{"cID": "9siogRgZo3Y", "title": "Kickflip", "cstart": "9656", "cend": "13143"},
{"cID": "9siogRgZo3Y", "title": "Kickflip", "cstart": "32169", "cend": "34800"},
{"cID": "9siogRgZo3Y", "title": "Kickflip", "cstart": "55760", "cend": "59292"},
{"cID": "9siogRgZo3Y", "title": "Kickflip", "cstart": "204266", "cend": "207530"},
{"cID": "9siogRgZo3Y", "title": "Kickflip", "cstart": "277463", "cend": "280556"}]
vids = [{"cID": "qZEElv92rLM", "title": "Onda Onda", "cstart": "35137", "cend": "42571"},
{"cID": "qZEElv92rLM", "title": "Controller Crew", "cstart": "110635", "cend": "112780"},
{"cID": "qZEElv92rLM", "title": "Hit It", "cstart": "120177", "cend": "132160"},
{"cID": "qZEElv92rLM", "title": "Billy Bounce", "cstart": "154580", "cend": "160941"},
{"cID": "qZEElv92rLM", "title": "Dont' Start Now", "cstart": "175213", "cend": "188833"},
{"cID": "qZEElv92rLM", "title": "Savage", "cstart": "197961", "cend": "211121"},
{"cID": "qZEElv92rLM", "title": "The Flow", "cstart": "221555", "cend": "231141"},
{"cID": "qZEElv92rLM", "title": "The Flow", "cstart": "233278", "cend": "238461"}]
import pafy, cv2, json, math
# url = 'https://youtu.be/DK797d9ozN0?t=440'

class Vid_Stream():
    def __init__(self,vids,output_file,options = None):
        self.yt_url = 'https://www.youtube.com/watch?v='
        self.vids = vids
        out = cv2.VideoWriter(output_file + '.mp4',cv2.VideoWriter_fourcc(*'MP4V'), 30, (640,480))
        if not options is None:
            self.options = json.loads(options)
        else:
            self.options = json.loads("""{"side":"both"}""")
            
            

        for vid in self.vids:
            vPafy = pafy.new(self.yt_url + vid['cID'])
            play = vPafy.getbest(preftype="mp4")
            cap = cv2.VideoCapture(play.url)
            cap.set(cv2.CAP_PROP_POS_MSEC, int(vid['cstart']))
            # Initiate holistic model
            with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:

                while cap.isOpened():
                    ret, frame = cap.read()
                    

                    # Recolor Feed
                    image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                           
                    height, width, channels = image.shape
                    if self.options['side'] == 'right':
                        image = image[:, math.floor(width/2):width,:]
                        cv2.imwrite('image.jpg',image)
#                     image.flags.writeable = False 
                    # Make Detections
                    results = holistic.process(image)
                    # print(results.face_landmarks)

                    # face_landmarks, pose_landmarks, left_hand_landmarks, right_hand_landmarks

                    # Recolor image back to BGR for rendering
                    image.flags.writeable = True   
                    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

            #         # 1. Draw face landmarks
            #         mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS, 
            #                                  mp_drawing.DrawingSpec(color=(80,110,10), thickness=1, circle_radius=1),
            #                                  mp_drawing.DrawingSpec(color=(80,256,121), thickness=1, circle_radius=1)
            #                                  )

                    # 2. Right hand
                    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 
                                             mp_drawing.DrawingSpec(color=(80,22,10), thickness=2, circle_radius=4),
                                             mp_drawing.DrawingSpec(color=(80,44,121), thickness=2, circle_radius=2)
                                             )

                    # 3. Left Hand
                    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 
                                             mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4),
                                             mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2)
                                             )

                    # 4. Pose Detections
                    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS, 
                                             mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4),
                                             mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2)
                                             )
                    # Export coordinates
                    try:
                            # Extract Pose landmarks
                        pose = results.pose_landmarks.landmark
                        pose_row = list(np.array([[landmark.x, landmark.y, landmark.z, landmark.visibility] for landmark in pose]).flatten())

            #                 # Extract Face landmarks
            #                 face = results.face_landmarks.landmark
            #                 face_row = list(np.array([[landmark.x, landmark.y, landmark.z, landmark.visibility] for landmark in face]).flatten())

                        # Concate rows
                        row = pose_row   #+face_row

                        # Append class name 
                        row.insert(0, vid['title'])

                        # Export to CSV
                        with open('coords.csv', mode='a', newline='') as f:
                            csv_writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
                            csv_writer.writerow(row)

                    except:
                        pass

                    cv2.imshow('Trick Maneuver Training video: ' + vid['title'], image)
                    # resize image
                    dim = (640, 480)
                    image = cv2.resize(image, dim, interpolation = cv2.INTER_AREA)
                    out.write(image)

                    if cv2.waitKey(10) & 0xFF == ord('q'):
                        break

                    cur_time = cap.get(cv2.CAP_PROP_POS_MSEC)
                    if cur_time > int(vid['cend']):
                        break

            cap.release()
            cv2.destroyAllWindows()
        out.release()

options="""{"side":"right"}"""
Vid_Stream(vids[:1],'output_file',options=options)<jupyter_output><empty_output><jupyter_text>2. Capture Landmarks & Export to CSV-->--><jupyter_code>import csv
import os
import numpy as np
num_coords = len(results.pose_landmarks.landmark)+len(results.face_landmarks.landmark)
landmarks = ['class']
for val in range(1, num_coords+1):
    landmarks += ['x{}'.format(val), 'y{}'.format(val), 'z{}'.format(val), 'v{}'.format(val)]
landmarks<jupyter_output><empty_output><jupyter_text>3. Train Custom Model Using Scikit Learn  3.1 Read in Collected Data and Process<jupyter_code>import pandas as pd
from sklearn.model_selection import train_test_split
df = pd.read_csv('coords.csv')
df.head()
df.tail()
df[df['class']=='Ollie']

X = df.drop('class', axis=1) # features
X.dropna(axis=1, how='any', inplace=True)
y = df['class'] # target value
df.shape
X.shape
X
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1234)
y_test<jupyter_output><empty_output><jupyter_text>3.2 Train Machine Learning Classification Model<jupyter_code>from sklearn.pipeline import make_pipeline 
from sklearn.preprocessing import StandardScaler 

from sklearn.linear_model import LogisticRegression, RidgeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
pipelines = {
    'lr':make_pipeline(StandardScaler(), LogisticRegression()),
    'rc':make_pipeline(StandardScaler(), RidgeClassifier()),
    'rf':make_pipeline(StandardScaler(), RandomForestClassifier()),
    'gb':make_pipeline(StandardScaler(), GradientBoostingClassifier()),
}
fit_models = {}
for algo, pipeline in pipelines.items():
    model = pipeline.fit(X_train, y_train)
    fit_models[algo] = model
fit_models
fit_models['rc'].predict(X_test)<jupyter_output><empty_output><jupyter_text>3.3 Evaluate and Serialize Model<jupyter_code>from sklearn.metrics import accuracy_score # Accuracy metrics 
import pickle 
for algo, model in fit_models.items():
    yhat = model.predict(X_test)
    print(algo, accuracy_score(y_test, yhat))
fit_models['rf'].predict(X_test)
y_test
with open('body_language.pkl', 'wb') as f:
    pickle.dump(fit_models['rf'], f)<jupyter_output><empty_output><jupyter_text>4. Make Detections with Model<jupyter_code>with open('body_language.pkl', 'rb') as f:
    model = pickle.load(f)
model
cap = cv2.VideoCapture(0)
# Initiate holistic model
with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
    
    while cap.isOpened():
        ret, frame = cap.read()
        
        # Recolor Feed
        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        image.flags.writeable = False        
        
        # Make Detections
        results = holistic.process(image)
        # print(results.face_landmarks)
        
        # face_landmarks, pose_landmarks, left_hand_landmarks, right_hand_landmarks
        
        # Recolor image back to BGR for rendering
        image.flags.writeable = True   
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        
        # 1. Draw face landmarks
        mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(80,110,10), thickness=1, circle_radius=1),
                                 mp_drawing.DrawingSpec(color=(80,256,121), thickness=1, circle_radius=1)
                                 )
        
        # 2. Right hand
        mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(80,22,10), thickness=2, circle_radius=4),
                                 mp_drawing.DrawingSpec(color=(80,44,121), thickness=2, circle_radius=2)
                                 )

        # 3. Left Hand
        mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4),
                                 mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2)
                                 )

        # 4. Pose Detections
        mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS, 
                                 mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4),
                                 mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2)
                                 )
        # Export coordinates
        try:
            # Extract Pose landmarks
            pose = results.pose_landmarks.landmark
            pose_row = list(np.array([[landmark.x, landmark.y, landmark.z, landmark.visibility] for landmark in pose]).flatten())
            
            # Extract Face landmarks
            face = results.face_landmarks.landmark
            face_row = list(np.array([[landmark.x, landmark.y, landmark.z, landmark.visibility] for landmark in face]).flatten())
            
            # Concate rows
            row = pose_row+face_row
            
#             # Append class name 
#             row.insert(0, class_name)
            
#             # Export to CSV
#             with open('coords.csv', mode='a', newline='') as f:
#                 csv_writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
#                 csv_writer.writerow(row) 

            # Make Detections
            X = pd.DataFrame([row])
            body_language_class = model.predict(X)[0]
            body_language_prob = model.predict_proba(X)[0]
            print(body_language_class, body_language_prob)
            
            # Grab ear coords
            coords = tuple(np.multiply(
                            np.array(
                                (results.pose_landmarks.landmark[mp_holistic.PoseLandmark.LEFT_EAR].x, 
                                 results.pose_landmarks.landmark[mp_holistic.PoseLandmark.LEFT_EAR].y))
                        , [640,480]).astype(int))
            
            cv2.rectangle(image, 
                          (coords[0], coords[1]+5), 
                          (coords[0]+len(body_language_class)*20, coords[1]-30), 
                          (245, 117, 16), -1)
            cv2.putText(image, body_language_class, coords, 
                        cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
            
            # Get status box
            cv2.rectangle(image, (0,0), (250, 60), (245, 117, 16), -1)
            
            # Display Class
            cv2.putText(image, 'CLASS'
                        , (95,12), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)
            cv2.putText(image, body_language_class.split(' ')[0]
                        , (90,40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
            
            # Display Probability
            cv2.putText(image, 'PROB'
                        , (15,12), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)
            cv2.putText(image, str(round(body_language_prob[np.argmax(body_language_prob)],2))
                        , (10,40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
            
        except:
            pass
                        
        cv2.imshow('Raw Webcam Feed', image)

        if cv2.waitKey(10) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()
tuple(np.multiply(np.array((results.pose_landmarks.landmark[mp_holistic.PoseLandmark.LEFT_EAR].x, 
results.pose_landmarks.landmark[mp_holistic.PoseLandmark.LEFT_EAR].y)), [640,480]).astype(int))<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup