Real-time transcription and sentiment analysis of audio streams; on the phone and in the browser

A presentation at Geektime Code in June 2018 in Tel Aviv-Yafo, Israel by Aaron Bassett

Slide 1

Slide 1

Slide 2

Slide 2

@aaronbassett

Slide 3

Slide 3

Slide 4

Slide 4

Slide 5

Slide 5

Slide 6

Slide 6

Slide 7

Slide 7

Slide 8

Slide 8

Slide 9

Slide 9

(function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() { ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());

Slide 10

Slide 10

(function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() { ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());

Slide 11

Slide 11

(function poll() { new Ajax.Request('/api/', { method:'get', onSuccess: function() { ... }, onFailure: function() { ... } }); setTimeout(poll, 1000); }());

Slide 12

Slide 12

(function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());

Slide 13

Slide 13

(function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());

Slide 14

Slide 14

(function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());

Slide 15

Slide 15

(function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());

Slide 16

Slide 16

(function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());

Slide 17

Slide 17

(function poll() { new Ajax.Request('/api/', { method: 'get', timeout: 60000, onSuccess: function() { // Do something poll(); }, onFailure: function() { // Do something else poll(); } }); }());

Slide 18

Slide 18

“typical header sizes of 700-800 bytes is common” –Google “SPDY: An experimental protocol for a faster web"

Slide 19

Slide 19

2 bytes. 0x00 UTF8 DATA 0xFF

Slide 20

Slide 20

100,000

Slide 21

Slide 21

700 Bytes

Slide 22

Slide 22

560,000,000

Slide 23

Slide 23

70MB/s

Slide 24

Slide 24

0.2MB/s

Slide 25

Slide 25

Slide 26

Slide 26

https://git.io/ fNlS2

Slide 27

Slide 27

http://werise-tornadoserver.ngrok.io/dashboard

Slide 28

Slide 28

Slide 29

Slide 29

write_message on_message

Slide 30

Slide 30

Slide 31

Slide 31

Slide 32

Slide 32

Slide 33

Slide 33

Slide 34

Slide 34

Slide 35

Slide 35

h s i

Slide 36

Slide 36

Slide 37

Slide 37

Slide 38

Slide 38

Slide 39

Slide 39

Slide 40

Slide 40

👍

Slide 41

Slide 41

Slide 42

Slide 42

var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);

Slide 43

Slide 43

var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);

Slide 44

Slide 44

var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);

Slide 45

Slide 45

var soundAllowed = function (stream) { window.persistAudioStream = stream; var audioContent = new AudioContext(); var audioStream = audioContent.createMediaStreamSource( stream ); } var soundNotAllowed = function (error) { h.innerHTML = "You must allow your microphone."; console.log(error); } navigator.getUserMedia({audio:true}, soundAllowed, soundNotAllowed);

Slide 46

Slide 46

Slide 47

Slide 47

Slide 48

Slide 48

Slide 49

Slide 49

Slide 50

Slide 50

Slide 51

Slide 51

👍

Slide 52

Slide 52

Slide 53

Slide 53

def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from': os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]

Slide 54

Slide 54

def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from': os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]

Slide 55

Slide 55

def proxy(self): return [ { 'action': 'connect', 'eventUrl': [f'{self.base_url}/events'], 'from': os.environ['NEXMO_NUMBER'], 'endpoint': [ { 'type': 'websocket', 'uri': f'{os.environ["WEBSOCKET_SERVER_URL"]}/socket', 'content-type': 'audio/l16;rate=16000', 'headers': {} } ] } ]

Slide 56

Slide 56

Slide 57

Slide 57

Slide 58

Slide 58

Slide 59

Slide 59

Slide 60

Slide 60

Slide 61

Slide 61

def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)

Slide 62

Slide 62

def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)

Slide 63

Slide 63

def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)

Slide 64

Slide 64

def on_message(self, message): transcriber = yield self.transcriber if type(message) != str: transcriber.write_message(message, binary=True) else: logger.info(message) data = json.loads(message) data['action'] = "start" data['continuous'] = True data['interim_results'] = True transcriber.write_message(json.dumps(data), binary=False)

Slide 65

Slide 65

self.transcriber = tornado.websocket.websocket_connect( 'wss://[url]?watson-token={token}&model={model}'.format( token=self.transcriber_token, model=os.environ['REALTIME_TRANSCRIBER_MODEL'] ), on_message_callback=self.on_transcriber_message )

Slide 66

Slide 66

self.transcriber = tornado.websocket.websocket_connect( 'wss://[url]?watson-token={token}&model={model}'.format( token=self.transcriber_token, model=os.environ['REALTIME_TRANSCRIBER_MODEL'] ), on_message_callback=self.on_transcriber_message )

Slide 67

Slide 67

Slide 68

Slide 68

Slide 69

Slide 69

Slide 70

Slide 70

Slide 71

Slide 71

Slide 72

Slide 72

Slide 73

Slide 73

def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))

Slide 74

Slide 74

def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))

Slide 75

Slide 75

def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))

Slide 76

Slide 76

def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))

Slide 77

Slide 77

def on_transcriber_message(self, message): if message: message = json.loads(message) if 'results' in message: transcript = message['transcript'] tone_results = self.tone_analyzer.tone( tone_input=transcript, content_type="text/plain" ) tones = tone_results['tones'] DashboardHandler.send_updates(json.dumps(tones))

Slide 78

Slide 78

Slide 79

Slide 79

Slide 80

Slide 80

Slide 81

Slide 81

class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass

Slide 82

Slide 82

class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass

Slide 83

Slide 83

class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass

Slide 84

Slide 84

class DashboardHandler(tornado.websocket.WebSocketHandler): waiters = set() def open(self): DashboardHandler.waiters.add(self) def on_close(self): DashboardHandler.waiters.remove(self) @classmethod def send_updates(cls, tones): for waiter in cls.waiters: try: waiter.write_message(tones) except: pass

Slide 85

Slide 85

Slide 86

Slide 86

Slide 87

Slide 87

var emotions = { anger: new TimeSeries(), disgust: new TimeSeries(), fear: new TimeSeries(), joy: new TimeSeries(), sadness: new TimeSeries() }

Slide 88

Slide 88

var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }

Slide 89

Slide 89

var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }

Slide 90

Slide 90

var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }

Slide 91

Slide 91

var websocket = new WebSocket('{{ server_url }}/dashboard-socket'); websocket.onmessage = function(evt) { JSON.parse(evt.data).map(function(emotion){ emotions[emotion.tone_id].append( new Date().getTime(), emotion.score ) }); }

Slide 92

Slide 92

Slide 93

Slide 93

Slide 94

Slide 94

Slide 95

Slide 95

@aaronbassett @nexmodev developer.nexmo.com github.com/nexmo-community