The Latest and
Coolest in
HTML5 Media

Or: WTF are browsers up to these days...

Use your cursor keys to move forward and backward through the slides.

Best viewed in Firefox 4beta

1
Overview

http://caniuse.com/

  • Video Manipulation
  • Audio Manipulation
  • Media Accessibility
  • it's not just two new tags in HTML...

2
Power of Combining Technologies

Definitive Guide to HTML5 video chapter index

3

Video Element & Video Manipulation

4
Innocent Web browser <video> element
              <video controls poster="HelloWorld.png" width="500">
                  <source src="HelloWorld.mp4" type="video/mp4">
                  <source src="HelloWorld.webm" type="video/webm">
                  <source src="HelloWorld.ogv" type="video/ogg">
              </video>
            
5
Well, not just HTML - here's the CSS
      video {
        border-radius: 12px;
        border: 1px solid rgba(0, 0, 0, 0.2);
      }
            
6
More CSS3: transitions
      video {                         video, video:hover {
        height:    200px;               position: relative;
        padding:    10px;               border:   5px solid black;
        left:       50px;               transition-property: all;
        top:        50px;               transition-duration: 0.5s;
        background-color: white;        transition-timing-function: linear;
      }                               }
      video:hover {
         height:   250px;
         padding:   15px;
         left:      25px;
         top:       25px;
         background-color: black;
       }
            
7
More cool CSS3: transforms (rotate)
8
CSS3 only in Safari: 3D

Video on the sides of a 3D cube


http://html5videoguide.net/code_c3_16.html

(Load in Safari)

9
Media and the JavaScript API
10
<video> and the JavaScript API
              video = document.getElementsByTagName("video")[0];
              
              // functions
              video.load();
              video.play();
              video.pause();
              
              // properties
              video.currentSrc
              video.currentTime
              video.duration
              
              // events
              loadedmetadata
              timeupdate
              pause
              play
              ended
            
11
Custom Player using JS and CSS
12
SVG and Video

Cool filters, but only work in Firefox with in-line SVG

13
Grabbing the video frame into a Canvas
<video controls height="270" width="480" >
  <source src="HelloWorld.mp4"  type="video/mp4">
  <source src="HelloWorld.webm" type="video/webm">
</video>
<canvas width="400" height="300" style="border: 1px solid black;">
</canvas>
<script>
  window.onload = function() {
    initCanvas();
  }
  var context;
  function initCanvas() {
    video = document.getElementsByTagName("video")[0];
    canvas = document.getElementsByTagName("canvas")[0];
    context = canvas.getContext("2d");
    video.addEventListener("timeupdate", paintFrame, false);
  }
  function paintFrame() {
    context.drawImage(video, 0, 0, 160, 80);
  }
</script>
14
Faster grabbing of frames
  function initCanvas() {
    video = document.getElementsByTagName("video")[0];
    canvas = document.getElementsByTagName("canvas")[0];
    context = canvas.getContext("2d");
    video.addEventListener("play", paintFrame, false);
  }
  function paintFrame() {
    context.drawImage(video, 0, 0, 160, 80);
    if (video.paused || video.ended) {
      return;
    }
    setTimeout(function () {
        paintFrame();
    }, 0);
  }
            
15
Processing Frame Data - a "scratch" canvas
        context.drawImage(video, 0, 0, w, h);
        frame = context.getImageData(0, 0, w, h);
        for (var i = 0; i < frame.data.length; i += 4) {
          r += frame.data[i];
          g += frame.data[i + 1];
          b += frame.data[i + 2];
        }
        ambience.style.backgroundColor = 'rgb('+r+','+g+','+b+')';
            
16
Pixel Manipulation
17
Pixel Manipulation - the code
    output = context.createImageData(w, h);

    // Loop over each pixel of input file
    for (x = 0; x < w; x++) {
      for (y = 0; y < h; y++) {
        // index in output image
        i = x + w*y;
        for (c = 0; c < 4; c++) {
          output.data[4*i+c] = frame.data[4*i+c];
        }
        // make pixels transparent
        r = frame.data[i * 4 + 0];
        g = frame.data[i * 4 + 1];
        b = frame.data[i * 4 + 2];
        if (!(r > 200 && g > 200 && b > 200))
          output.data[4*i + 3] = 0;
      }
    }

    // Draw the ImageData object.
    context.putImageData(output, 0, 0);
            
18
<video> and Canvas - reflection
              rctxt.translate(0,160);
              rctxt.scale(1,-1);
              gradient = rctxt.createLinearGradient(0, 105, 0, 160);
              gradient.addColorStop(1, "rgba(255, 255, 255, 0.3)");
              gradient.addColorStop(0, "rgba(255, 255, 255, 1.0)");
              rctxt.fillStyle = gradient;
              rctxt.drawImage(video, 0, 0, 320, 160);
            
19
<video> and Web Workers - face detection
    function isSkin(rn, gn, bn, base) {
      if (rn > 0.35 && rn < 0.5 && gn > 0.2 && gn < 0.5 && bn > 0.2 && bn < 0.35 
                    && base > 250) {
        return true;
      } else {
        return false;
      }
    }
            
20
Shared Video Viewing

Try to connect to http://169.222.9.130/code_c10_6.html

Make sure to run a Web Browser that supports Web Sockets: http://websocket.org/

    var socket = new WebSocket('ws://169.222.9.130:8080/');
    socket.addEventListener("message", onMessage, false);
    video.addEventListener("play", sendPlay, false);
    function sendPlay() {
      socket.send("play ");
    }
    function onMessage(evt) {
      output.innerHTML += "RECEIVED: "+evt.data+"<br/>";
    }
           
21

Audio Element & Audio Manipulation

22
Innocent browser <audio> element
              <audio controls>
                  <source src="revolve.mp3" type="audio/mp4">
                  <source src="revolve.ogg" type="audio/ogg">
              </audio>
            
23
Mozilla Audio API show-off
Get Firefox 4 beta

http://videos.mozilla.org/serv/blizzard/audio-slideshow/
24
How to do this yourself - quick intro

(Firefox4 only)

  <audio id="audio_samples" src="revolve.ogg" controls>
  </audio>

  <script type="text/javascript">
    var audio = document.getElementById("audio_samples");

    audio.addEventListener("loadedmetadata", getMetadata, false);

    function getMetadata (event) {
      channels = audio.mozChannels;
      rate     = audio.mozSampleRate;
      fbLength = audio.mozFrameBufferLength;
    }
    
    audio.addEventListener("MozAudioAvailable", writeSamples, false);

    function writeSamples (event) {
      var data = event.frameBuffer;
      var frame_start = event.time;
    }
  </script>
            
25
Generating Sound with the Audio Data API
    var audio = new Audio();
    rate = 44100;
    audio.mozSetup(1, rate);

    // set up sample array of size 500ms
    var samples = new Float32Array(22050);

    var k = 2* Math.PI * frequency / rate;
    for (var i=0, size=samples.length; i < size; i++) {
      samples[i] = Math.sin(k * currentSoundSample++);
    }
    
    audio.mozWriteAudio(samples);
            

(Firefox4 only)

26
Web Audio API Standardisation

Audio Filter Graph

27

Media Accessibility

28
Key Focus Media Accessibility
  • Captions and Subtitles
  • Audio Descriptions
  •  
  • External Text Files
  • In-band content
29
Three key specifications
30
Captions, Subtitles, Text Descriptions
WEBVTT FILE

1
00:00:08,124 --> 00:00:10,742
Workstations and high end personal computers have been able to

2
00:00:10,742 --> 00:00:14,749
manipulate digital audio pretty easily for about fifteen years now.

3
00:00:14,749 --> 00:00:17,470
It's only been about five years that a decent workstation's been able

4
00:00:17,470 --> 00:00:21,643
to handle raw video without a lot of expensive special purpose hardware.
            
31
Example with Subtitles in JavaScript
32
Internationalized Subtitles
WEBVTT FILE

00:00:15.000 --> 00:00:17.950 A:end D:vertical
在左边我们可以看到...

00:00:18.160 --> 00:00:20.080 A:end D:vertical
在右边我们可以看到...

00:00:20.110 --> 00:00:21.960 A:end D:vertical S:70% L:40%
...捕蝇草械.
            
33
Karaoke-Style Paint-On Subtitles
WEBVTT FILE

1
00:00:10.000 --> 00:00:12.210
<00:00:10.035>Chocolate <00:00:11.000>Rain

2
00:00:12.210 --> 00:00:15.910
<00:00:13.250>Some <00:00:13.500>stay <00:00:13.750>dry <00:00:14.25>and 
<00:00:14.50>others <00:00:15.00>feel <00:00:15.25>the <00:00:15.50>pain

3
00:00:15.910 --> 00:00:15.920
<00:00:16.000>Chocolate <00:00:16.500>Rain
          
34
Markup

The <track> Element

<video src="media/sintel.ogv" poster="media/poster_sintel.png" controls width="50%">

  <track kind="subtitles" src="text/vid1-en.vtt" srclang="en" label="English" default>
  <track kind="subtitles" src="text/vid1-fr.vtt" srclang="fr" label="Français">
  <track kind="subtitles" src="text/vid1-de.vtt" srclang="de" label="Deutsch">

  <track kind="captions" src="text/vid1-cap.vtt" srclang="en" label="English">

  <track kind="descriptions" src="text/vid1-desc.vtt" srclang="en" label="English">

  <track kind="chapters" src="text/vid1-chap.vtt" srclang="en" label="English">

  <track kind="metadata" src="text/vid1-json.vtt" srclang="en" label="English">

</video>
            
35
TextTrack API
  interface TextTrack {
    readonly attribute DOMString kind;
    readonly attribute DOMString label;
    readonly attribute DOMString language;

    readonly attribute unsigned short readyState; // LOADING/LOADED/ERROR

    readonly attribute TextTrackCueList cues;
    readonly attribute TextTrackCueList activeCues;

    readonly attribute Function oncuechange;
  };
            
36
CSS Styling
  • Pseudo-element: ::cue
  • ::cue-part(arg) for matching cue parts
    • voice: a voice identifier
    • part: a cue part one of <i>, <b>, <ruby>, <rt>
    • position: 'past' or 'future' relative to a cue timestamp
37
Next Steps
  • WebVTT format improvements
  • JS implementations
  • Browser Vendors to support markup and format
38
More Details: Shameless Plug

Definitive Guide to HTML5 video book cover

For the details, see my book:

The Definitive Guide to HTML5 Video

39
Thank you
Questions?

              Silvia Pfeiffer
              blog.gingertech.net
              silviapfeiffer1@gmail.com
              @silviapfeiffer
            
Thanks go to the Blender organisation and to Creative Commons for their videos.
40