Added backend file to lang_learn for short video clips project

backend/app/analysis/short_video_clips.py

@@ -0,0 +1,96 @@
+"""
+Helper functions used to get information of videos from URLs such as its 
+transcript and title
+"""
+import isodate
+import os
+import time
+
+import googleapiclient.discovery
+from youtube_transcript_api import YouTubeTranscriptApi
+
+api_service_name = "youtube"
+api_version = "v3"
+DEVELOPER_KEY = "AIzaSyCwwNqx7_HiX_KvxOGckFI9JxMy7LqdK2s"
+
+youtube = googleapiclient.discovery.build(
+    api_service_name, api_version, developerKey = DEVELOPER_KEY)
+
+def get_video_data(video_id):
+    """
+    Given an id of a video, gets its title and transcript in an interval format
+
+    :param video_id: ID of video
+    :return: A dictionary with title and transcript of video with ID video_id
+    """
+    transcript = YouTubeTranscriptApi.get_transcript(video_id)
+    request = youtube.videos().list(
+        id=video_id,
+        part="snippet,contentDetails",
+        fields="items/snippet/title,items/contentDetails/duration",
+    )
+    response_data = request.execute()['items'][0]
+    title = response_data['snippet']['title']
+    duration = response_data['contentDetails']['duration']
+
+    timedelta_dur = isodate.parse_duration(duration)
+    dur_seconds = int(timedelta_dur.total_seconds())
+    end_timestamp = timestamp_format(dur_seconds)
+
+    return { 
+        'title': title,
+        'transcript': transcript_intervals_format(transcript, end_timestamp),
+    }
+
+
+def create_transcript_snippet(start, end, text):
+    """
+    Generate a snippet given the start, end, and text of a transcript interval
+
+    :param start: Start time of transcript interval
+    :param end: End time of transcript interval
+    :param text: Text content of transcript interval
+    :return: A dictionary with the start/end times and text content of transcript interval
+    """
+    return {
+            'start': start,
+            'end': end,
+            'text': text,
+        }
+
+
+def transcript_intervals_format(transcript, end_timestamp):
+    """
+    Given a video transcript and last timestamp, format transcript content into intervals
+
+    :param transcript: Video transcript
+    :param end_timestamp: Last timestamp on video
+    :return: List of dictionaries that represent an interval of video transcript
+    """
+    intervals = []
+    for i in range(len(transcript)-1):
+        start, end = round(transcript[i]['start']), round(transcript[i+1]['start'])
+        text = transcript[i]['text']
+        snippet = create_transcript_snippet(
+            timestamp_format(start), timestamp_format(end), text
+        )
+        intervals.append(snippet)
+
+    # Add final interval through YouTube API call because duration is 
+    # not accurate from captions transcript through the "dur" attribute
+    prev_end = intervals[-1]['end']
+    end_text = transcript[-1]['text']
+    end_snippet = create_transcript_snippet(prev_end, end_timestamp, end_text)
+    intervals.append(end_snippet)
+
+    return intervals
+
+
+def timestamp_format(duration):
+    """
+    Given a duration in seconds, formats time to MM:SS
+
+    :param duration: duration of
+    :return: String of time duration formatted as MM:SS
+    """
+    return time.strftime('%M:%S', time.gmtime(duration))

backend/app/views.py

@@ -34,6 +34,8 @@
 )
 from .quiz_creation.conjugation_quiz import get_quiz_sentences
 
+from .analysis.short_video_clips import get_video_data
+
 
 @api_view(['GET'])
 def all_text(request):
@@ -243,3 +245,9 @@ def get_response_quiz_data(request, text_id):
         raise Http404 from text_not_exist
     res = get_quiz_questions(text_obj.content)
     return Response(res)
+
+@api_view(['GET'])
+def get_video_transcript(request, video_id):
+    transcript = get_video_data(video_id)
+    return Response(transcript)
+

backend/config/urls.py

@@ -29,6 +29,7 @@
     get_crossword,
     get_quiz_data,
     text, get_response_quiz_data,
+    get_video_transcript,
 )
 
 
@@ -61,6 +62,7 @@ def react_view_path(route, component_name):
     path('api/text/<int:text_id>', text),
     path('api/get_picturebook_prompt/<int:text_id>/<str:part_of_speech>', get_picturebook_prompt),
     path('api/get_picturebook_data', get_picturebook_data),
+    path('api/get_video_transcript/<str:video_id>', get_video_transcript),
 
     # View paths
     react_view_path('', 'IndexView'),

frontend/package.json

@@ -64,7 +64,8 @@
     "webpack-bundle-tracker": "^0.4.2-beta",
     "webpack-dev-server": "^3.11.0",
     "webpack-manifest-plugin": "2.0.4",
-    "workbox-webpack-plugin": "4.2.0"
+    "workbox-webpack-plugin": "4.2.0",
+    "youtube-captions-scraper": "^1.0.3"
   },
   "scripts": {
     "start": "node scripts/start.js",

frontend/src/app.js

@@ -8,18 +8,18 @@ import ReactDOM from 'react-dom';
 import 'bootstrap/dist/css/bootstrap.min.css';
 
 import { IndexView } from './index/index';
-import { QuizView } from './quizView/quizView';
-import { AnagramView } from './anagramView/anagramView';
-import { InstructorView } from './instructorView/instructorView';
-import { FlashcardView } from './flashcard/flashcardView';
-import { AllQuizView } from './quizView/allQuizView';
-import { PictureBookView } from './pictureBookView/pictureBookView';
-import { CrosswordView } from './crosswordView/crosswordView';
-import { ResponseAllQuizView } from './responseQuizView/responseAllQuizView';
-import { ResponseQuizView } from './responseQuizView/responseQuizView';
+import { QuizView } from './views/quiz/quizView';
+import { AnagramView } from './views/anagram/anagramView';
+import { InstructorView } from './views/instructor/instructorView';
+import { FlashcardView } from './views/flashcard/flashcardView';
+import { AllQuizView } from './views/quiz/allQuizView';
+import { PictureBookView } from './views/pictureBook/pictureBookView';
+import { CrosswordView } from './views/crossword/crosswordView';
+import { ResponseAllQuizView } from './views/responseQuiz/responseAllQuizView';
+import { ResponseQuizView } from './views/responseQuiz/responseQuizView';
 
 // Import all styles
-import './UILibrary/styles.scss';
+import './components/styles.scss';
 
 window.app_modules = {
     React,  // Make React accessible from the base template

frontend/src/UILibrary/styles.scss → frontend/src/components/styles.scss

@@ -3,11 +3,11 @@
  */
 @import "../../node_modules/bootstrap/scss/bootstrap";
 @import '../index/index';
-@import '../anagramView/anagramView';
-@import '../crosswordView/crosswordView';
-@import '../instructorView/instructorView';
-@import '../flashcard/flashcardView';
-@import '../pictureBookView/pictureBookView';
+@import '../views/anagram/anagramView';
+@import '../views/crossword/crosswordView';
+@import '../views/instructor/instructorView';
+@import '../views/flashcard/flashcardView';
+@import '../views/pictureBook/pictureBookView';
 
 html {
     position: relative;

frontend/src/index/index.js

@@ -1,6 +1,6 @@
 import React from 'react';
 import * as PropTypes from 'prop-types';
-import { Navbar, Footer } from '../UILibrary/components';
+import { Navbar, Footer } from '../components/components';
 
 
 // Functions to use to generate links for different formats

frontend/src/anagramView/anagramView.js → frontend/src/views/anagram/anagramView.js

@@ -3,7 +3,7 @@ import ReactTooltipDefaultExport from 'react-tooltip';
 import Confetti from 'react-dom-confetti';
 import * as PropTypes from 'prop-types';
 
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
 
 const CONFETTI_CONFIG = {
     angle: 90,

frontend/src/crosswordView/crosswordView.js → frontend/src/views/crossword/crosswordView.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import * as PropTypes from 'prop-types';
 
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
 import Crossword from './crossword';
 
 // Given a word and the clue that it is part of, remove the word from the clue

frontend/src/flashcard/flashcardView.js → frontend/src/views/flashcard/flashcardView.js

@@ -1,7 +1,7 @@
 import React, { Component } from 'react';
 import * as PropTypes from 'prop-types';
 
-import { Footer, Navbar, LoadingPage } from '../UILibrary/components';
+import { Footer, Navbar, LoadingPage } from '../../components/components';
 
 const capitalize = (word) => {
     return word.charAt(0).toUpperCase() + word.slice(1);

frontend/src/instructorView/instructorView.js → frontend/src/views/instructor/instructorView.js

@@ -1,8 +1,8 @@
 import React from 'react';
 import * as PropTypes from 'prop-types';
 
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
-import { capitalize, getCookie } from '../common';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
+import { capitalize, getCookie } from '../../common';
 
 class Module extends React.Component {
     render() {

frontend/src/pictureBookView/pictureBookView.js → frontend/src/views/pictureBook/pictureBookView.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import * as PropTypes from 'prop-types';
 
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
 
 const capitalize = (word) => {
     return word.charAt(0).toUpperCase() + word.slice(1);

frontend/src/quizView/allQuizView.js → frontend/src/views/quiz/allQuizView.js

@@ -3,7 +3,7 @@ import './quizView.scss';
 // import ReactTooltipDefaultExport from 'react-tooltip';
 // import * as PropTypes from 'prop-types';
 import { Button } from 'react-bootstrap';
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
 
 export class AllQuizView extends React.Component {
     constructor(props) {

frontend/src/quizView/quizView.js → frontend/src/views/quiz/quizView.js

@@ -8,7 +8,7 @@ import {
     ToggleButtonGroup,
 } from 'react-bootstrap';
 
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
 
 
 export class QuizView extends React.Component {

frontend/src/responseQuizView/responseAllQuizView.js → frontend/src/views/responseQuiz/responseAllQuizView.js

@@ -3,7 +3,7 @@ import './responseQuizView.scss';
 // import ReactTooltipDefaultExport from 'react-tooltip';
 // import * as PropTypes from 'prop-types';
 import { Button } from 'react-bootstrap';
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
 
 export class ResponseAllQuizView extends React.Component {
     constructor(props) {

frontend/src/responseQuizView/responseQuizView.js → frontend/src/views/responseQuiz/responseQuizView.js

@@ -10,7 +10,7 @@ import {
     ToggleButtonGroup,
 } from 'react-bootstrap';
 
-import { Navbar, Footer, LoadingPage } from '../UILibrary/components';
+import { Navbar, Footer, LoadingPage } from '../../components/components';
 
 // This dictionary maps specific error types to their respective error message.
 const ERROR_DESCRIPTIONS = {

requirements.txt

@@ -9,17 +9,19 @@ django-cors-headers==3.6.0
 django-webpack-loader==0.7.0
 djangorestframework==3.12.2
 futures==3.1.1
+google-api-python-client==2.27.0
 goslate==1.5.1
 idna==2.10
+isodate==0.6.0
 joblib==1.0.1
 nltk==3.5
 PyDictionary==2.0.1
 pyspellchecker==0.6.1
 pytz==2020.5
 regex==2020.11.13
 requests==2.25.1
+similar-sounding-words==0.1.2
 soupsieve==2.2
 sqlparse==0.4.1
 tqdm==4.58.0
-urllib3==1.26.3
-similar-sounding-words==0.1.2
+youtube-transcript-api==0.4.1