Load packages

library(data.table)
library(aws.s3)
library(aws.transcribe)
library(rjson)

We use data.table for some data wrangling, aws.3 & aws.transcribe to communicate with aws and rjson to handle json. Thanks to all package developers!

List files & upload

#define file path where your mp3 live
#replace "yourpath" with your path
file_path <- file.path("yourpath")

#List files mp3 in your folder

path_mp3  <- list.files(file_path, 
                        pattern = ".mp3",
                        full.names = TRUE)
#basename returns only the file name
files_mp3 <- basename(path_files_mp3)

Now we upload the mp3 to a s3-bucket

#define key and secret for your s3 bucket
#Do not store your key & secret this way

key    <- "yourkey"
secret <- "yoursecret"

#upload files to s3 bucket

for(i in seq_along(path_mp3)) {
  print(i)
  aws.s3::put_object(file = path_mp3[i], 
                     object = files_mp3[i], 
                     bucket = "yourbucket", 
                     key = key, 
                     secret = secret)
}

Start a transcribe job

First we need to understand what the package aws.transcribe does. We need the start_transcription function

aws.transcribe::start_transcription

## function (name, url, format = tools::file_ext(url), language = "en-US", 
##     hertz = NULL, ...) 
## {
##     bod <- list(Media = list(MediaFileUri = url))
##     bod$MediaFormat <- format
##     bod$LanguageCode <- language
##     if (!is.null(hertz)) {
##         bod$MediaSampleRateHertz <- hertz
##     }
##     bod$TranscriptionJobName <- name
##     transcribeHTTP(action = "StartTranscriptionJob", body = bod, 
##         ...)
## }
## <bytecode: 0x00000000154107d8>
## <environment: namespace:aws.transcribe>

The main takeaway here is that we define a list bod and then call the transcribeHTTP function. We can check the Amazon Transcribe documentation to understand, how we should define a request body which is called bod in this function.

https://docs.aws.amazon.com/de_de/transcribe/latest/dg/API_StartTranscriptionJob.html

Since I want to define the OutputBucketName, because we are going to write the actual transcribes (json) to a specific bucket, simply add an argument to the start_transcription function. I call the new function StartTranscription.

StartTranscription <- function (name, url, format = tools::file_ext(url), language = "en-US", 
          hertz = NULL, outputbucketname = "yourbucketname", ...) 
{
  bod <- list(Media = list(MediaFileUri = url))
  bod$MediaFormat <- format
  bod$LanguageCode <- language
  bod$OutputBucketName <- outputbucketname # added
  if (!is.null(hertz)) {
    bod$MediaSampleRateHertz <- hertz
  }
  bod$TranscriptionJobName <- name
  transcribeHTTP(action = "StartTranscriptionJob", body = bod, 
                 ...)
}

The transcribeHTTP function is a bit more complex.

aws.transcribe::transcribeHTTP

## function (action, query = list(), body = NULL, version = "v1", 
##     region = NULL, key = NULL, secret = NULL, session_token = NULL, 
##     ...) 
## {
##     d_timestamp <- format(Sys.time(), "%Y%m%dT%H%M%SZ", tz = "UTC")
##     if (is.null(region) || region == "") {
##         region <- "us-east-1"
##     }
##     url <- paste0("https://transcribe.", region, ".amazonaws.com")
##     Sig <- signature_v4_auth(datetime = d_timestamp, region = region, 
##         service = "transcribe", verb = "POST", action = "/", 
##         query_args = query, canonical_headers = list(host = paste0("transcribe.", 
##             region, ".amazonaws.com"), `x-amz-date` = d_timestamp, 
##             `X-Amz-Target` = paste0("Transcribe.", action), `Content-Type` = "application/x-amz-json-1.1"), 
##         request_body = if (is.null(body)) 
##             ""
##         else toJSON(body, auto_unbox = TRUE), key = key, secret = secret, 
##         session_token = session_token)
##     headers <- list()
##     headers[["X-Amz-Target"]] <- paste0("Transcribe.", action)
##     headers[["Content-Type"]] <- "application/x-amz-json-1.1"
##     headers[["x-amz-date"]] <- d_timestamp
##     headers[["x-amz-content-sha256"]] <- Sig$BodyHash
##     if (!is.null(session_token) && session_token != "") {
##         headers[["x-amz-security-token"]] <- session_token
##     }
##     headers[["Authorization"]] <- Sig[["SignatureHeader"]]
##     H <- do.call(add_headers, headers)
##     if (length(query)) {
##         r <- POST(url, H, query = query, body = body, encode = "json", 
##             ...)
##     }
##     else {
##         r <- POST(url, H, body = body, encode = "json", ...)
##     }
##     if (http_error(r)) {
##         x <- fromJSON(content(r, "text", encoding = "UTF-8"))
##         warn_for_status(r)
##         h <- headers(r)
##         out <- structure(x, headers = h, class = "aws_error")
##         attr(out, "request_canonical") <- Sig$CanonicalRequest
##         attr(out, "request_string_to_sign") <- Sig$StringToSign
##         attr(out, "request_signature") <- Sig$SignatureHeader
##     }
##     else {
##         out <- try(fromJSON(content(r, "text", encoding = "UTF-8")), 
##             silent = TRUE)
##         if (inherits(out, "try-error")) {
##             out <- structure(content(r, "text", encoding = "UTF-8"), 
##                 "unknown")
##         }
##     }
##     return(out)
## }
## <bytecode: 0x0000000015cdebc8>
## <environment: namespace:aws.transcribe>

We call this function inside StartTranscription with action = "StartTransciptionJob and body = bod. We use the ... argument to pass our secret and key, which are then passed to the POST function from the httr package. If you want more information on how to send requests to aws, please read up on https://docs.aws.amazon.com/de_de/transcribe/latest/dg/CommonParameters.html and https://docs.aws.amazon.com/de_de/general/latest/gr/signature-version-4.html. Thanks to Thomas J. Leeper (aws.s3 & aws.transcribe) that we do not have to fiddle with that for now!

Now I am defining a url, which points to the s3 bucket where I want to save the transcribes as json.

full_url <- paste0("https://s3.eu-central-1.amazonaws.com/yourbucket/", files_mp3)

for(i in seq_along(full_url)) {

StartTranscription(name = files_mp3[i], 
                   language = "de-DE", 
                   url = full_url[i], 
                   key = "yourkey",
                   secret = "yoursecret", 
                   region = "eu-central-1"
                   )
  
Sys.sleep(time = 60)
paste(i)
  
}

Download the json files

I actually need VTT files, because most video editing software are using this file type to include transcriptions. I wrote three helper functions to deal with downloading, formatting the json files and generating VTT files.

DownloadTransJson <- function(){
  list_bucket <- data.table::rbindlist(aws.s3::get_bucket(
    bucket = "yourbucket", 
    key = "yourkey", 
    secret = "yoursecret"))
  #only the json files
  list_bucket <- grep(x = list_bucket$Key, 
                      pattern = "mp3.json", 
                      value = TRUE)
  #download files in list_bucket
  for(i in list_bucket) {
  json_object    <- aws.s3::get_object(object =i,
                                       bucket = "yourbucket", 
                                       key = "yourkey", 
                                       secret = "yoursecret")
  formatted_json <- FormatJsonObject(jsonobject = json_object)
  WriteVTT(i, formatted_json)
  }
}

FormatJsonObject <- function(jsonobject){
  trans_00 <- rjson::fromJSON(rawToChar(jsonobject))
  
  transcript <- trans_00$results$transcripts[[1]]$transcript
  #split words
  words      <- unlist(strsplit(transcript, split = " "))
  
  #bind the list items
  timing_dt <- rbindlist(trans_00$results$items, 
                         fill = TRUE)
  #Getting rid of punctuation for now
  timing_dt <- timing_dt[type != "punctuation"]
  timing_dt[, word := words]
  
  #simple conversion to numeric
  timing_dt[, start_time := as.numeric(start_time)]
  timing_dt[, end_time := as.numeric(end_time)]
  
  #define a group for every ten words
  #so that ten words form a group
  timing_dt[, group := (1:.N - 1) %/% 10]
  
  #Collapse word to a phrase by group
  timing_dt[, phrase := lapply(.SD, paste, collapse = " "), 
            by = group, .SDcols = "word"]
  
  #get min start_time and max start time for a phrase
  timing_dt[, `:=` (min_start_time = min(start_time), max_end_time = max(end_time)), 
            by = "group"]
  
  #Need only one phrase per group, as they are redundant.
  unique_dt <- timing_dt[, .SD[1], by = "group"]
  
  unique_dt <- unique_dt[,.SD, .SDcols = c("phrase", 
                                           "min_start_time", 
                                           "max_end_time")]
  
  #
  unique_dt[, time_stamp := paste(paste0(as.ITime(min_start_time),",000"), 
                                  paste0(as.ITime(max_end_time),",000"), 
                                  sep = " --> ")]
  return(unique_dt)
}

WriteVTT <- function(name, jsondt) {
  
  file.create(paste0(name,".VTT"))
  con_file <- file(paste0(name,".VTT"))
  ##writing to VTT
  #might be os specific
  writeLines(paste0(1 : nrow(jsondt),"\n",
                    jsondt$time_stamp,"\n",
                    jsondt$phrase,"\n"), 
                    con_file)
}

DownloadTransJson()