wiki:MURML

Version 78 (modified by welberge, 6 years ago) (diff)

--

MURML

AsapRealizer's MURML extension encompasses the detailed description of facial animation and gesture. It does not encompas MURML's synchronization mechanisms and speech description, since these can be specified in BML and its core description levels. The namespace for MURML is  http://www.techfak.uni-bielefeld.de/ags/soa/murml.

MURML hand gesture

A hand gesture can be specified using the <murmlgesture> behavior.

<bml xmlns="http://www.bml-initiative.org/bml/bml-1.0"  id="bml1" > 
<murmlgesture id="gesture1" start="2" xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml"> 
  <murml-description>    
    <parallel>
      <dynamic slot="HandLocation" scope="left_arm">
	<dynamicElement type="linear" scope="left_arm">
          <value type="start" name="LocLowerChest LocCCenter LocNorm"/>
          <value type="end" name="LocStomach LocPeripheryLeft LocFFar"/>
	</dynamicElement>
      </dynamic>
      <dynamic slot="HandShape" scope="left_arm">
        <dynamicElement>
            <value type="start" name="BSneutral"/>
            <value type="end" name="BSflat"/>
	</dynamicElement>
      </dynamic>
      <dynamic slot="ExtFingerOrientation" scope="left_arm">
	<dynamicElement>
            <value type="start" name="DirRA"/>
            <value type="end" name="DirALAA"/>
	</dynamicElement>
      </dynamic>
      <dynamic slot="PalmOrientation" scope="left_arm">
	<dynamicElement>
            <value type="start" name="PalmRU"/>
            <value type="end" name="PalmU"/>
	</dynamicElement>
      </dynamic>
    </parallel> 
  </murml-description>
</murmlgesture> 
</bml> 

or, preferably, using the MURML description extension:

<gesture id="g1" mode="LEFT_HAND" lexeme="BEAT" start="0.5"> 
  <description priority="1" type="murmlgesture"> 
    <murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml"> 
    ...
    </murml-description> 
  </description> 
</gesture> 

MURML face gesture

MURML face gestures can similarly be described by the <murmlface> element

<bml xmlns="http://www.bml-initiative.org/bml/bml-1.0"  id="bml1" > 
<murmlface xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml" id="face1" start="2"> 
<murml-description>
<dynamic> 
	<keyframing easescale="5" easeturningpoint="0.7"> 
		<phase> 
		<frame ftime="0.5"> 
		<posture>Humanoid (Body_NG-mesh-morpher-yeux_NG01-1 1 100)</posture> 
		</frame> 
		<frame ftime="1"> 
		<posture>Humanoid (Body_NG-mesh-morpher-yeux_NG01-1 1 0)</posture> 
		</frame> 
		<frame ftime="1.5"> 
		<posture>Humanoid (Body_NG-mesh-morpher-yeux_NG01-1 1 100)</posture> 
		</frame> 
		<frame ftime="2"> 
		<posture>Humanoid (Body_NG-mesh-morpher-yeux_NG01-1 1 0)</posture> 
		</frame> 
		</phase> 
	</keyframing> 
</dynamic> 
</murmlface> 
</murml-description>
</bml> 

Or preferably as an extension of e.g. <faceLexeme>:

<faceLexeme id="face1" start="2" lexeme="smile"> 
	<description priority="1" type="murmlface"> 
	<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">	 
	<constraints>	 
	<dynamic> 
		<keyframing>			 
			<phase>			 
			<frame ftime="0.5"> 
			<posture>Humanoid (Body_NG-mesh-morpher-yeux_NG01-1 1 100)</posture> 
			</frame> 
			<frame ftime="1"> 
			<posture>Humanoid (Body_NG-mesh-morpher-yeux_NG01-1 1 0)</posture> 
			</frame>			 
			</phase> 
		</keyframing> 
	</dynamic> 
	</murml-description> 
	</description> 
</faceLexeme> 

From MURML to BML, an example

The example below shows how a MURML description can be translated to BML: MURML

<definition>
  <utterance>
    <specification>
    And now take <time id=“t1”/> this <time id=”t2”/> bar <time id=“t3”' chunkborder=“true”/>
    and make it <time id=“t4”/> this big. <time id=“t5”/>
    </specification>
    <focus onset=”t1” end=”t2” accent=”H*”/>
    <behaviorspec id=”gesture_1”>
      <gesture id=”pointing_to”>
        <affiliate onset=”t1” end=”t3”/>
        <param name=”refloc” value=”$Loc-Bar_1”/>
      </gesture>
    </behaviorspec>
    <behaviorspec id=”gesture_2”>
      <gesture>
        <affiliate onset=”t4” end=“t5”/>
        <constraints>
          <symmetrical dominant=”right_arm” symmetry=”SymMS”>
          <parallel>
            <static slot="HandShape" value="BSflat (FBround all o) (ThCpart o)"/>
            <static slot="PalmOrientation" value="DirL"/>
            <static slot="ExtFingerOrientation" value="DirA"/>
            <dynamic slot="HandLocation">
              <dynamicElement type="linear">
                <value type="start" name="LocShoulder LocCenterRight LocNorm"/>
                <value type="direction" name="DirR''/>
                <value type="distance" name="125.0''/>
              </dynamicElement>
            </dynamic>
          </parallel>
          </symmetrical>
        </constraints>
      </gesture>
    </behaviorspec>
  </utterance>
</definition>

BML

<bml id="bml1" xmlns="http://www.bml-initiative.org/bml/bml-1.0">
  <speech id="speech1"/>
    <text>
	And now take <sync id="t1"/> this <sync id="t2"/> bar
    </text>
    <description priority="1" type="maryxml">
      <maryxml xmlns="http://mary.dfki.de/2002/MaryXML">
        And now take <mark name="t1"/> <t accent="H*">this</t> <mark name="t2"/> bar
      </maryxml>    
    </description>
  </speech>  
  <pointing id="pointing_to" target="Loc-Bar_1" strokeStart="t1" strokeEnd="t3"/>
</bml>
<bml id="bml2" xmlns="http://www.bml-initiative.org/bml/bml-1.0" compositions="CHUNK_AFTER(bml1)">
  <speech id="speech1"/>
    <text>
    	and make it <sync id="t4"/> this big. <sync id="t5"/>
    </text>
  </speech>  
  <gesture id="gesture_2" strokeStart="t4" strokeEnd="t5" lexeme="SIZE" mode="BOTH_HANDS">
    <description priority="1" type="murmlgesture">
       <murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml"> 
       <symmetrical dominant="right_arm" symmetry="SymMS">
         <parallel>
          <static slot="HandShape" value="BSflat (FBround all o) (ThCpart o)"/>
          <static slot="PalmOrientation" value="DirL"/>
          <static slot="ExtFingerOrientation" value="DirA"/>
          <dynamic slot="HandLocation">
            <dynamicElement type="linear">
              <value type="start" name="LocShoulder LocCenterRight LocNorm"/>
              <value type="direction" name="DirR"/>
              <value type="distance" name="125.0"/>
            </dynamicElement>
          </dynamic>
         </parallel>
       </symmetrical>
       </murml-description>
    </description>
  </gesture>
</bml>

The MURML example contains two chunks, one with the intonation phrase "And now take this bar" + gesture gesture_1 and one with the intonation phrase "and make it this big." + gesture gesture_2. In BML, these chunks can be modeled in two BML blocks that are composed using the CHUNK_AFTER composition. Since core-bml does not contain any specification mechanisms for the MURML focus, we model the focus in speech using the  Mary XML core description extension. The MURML pointing gesture (with id gesture_1) can directly be translated into a BML pointing gesture. gesture_2 is modeled in core BML as a lexicalized gesture. Its finer grained MURML specification is provided within a description extension.

MURML descriptions in detail

<murml-description>

The toplevel element of a murml description is always <murml-description>. It acts on the same level as <constraints> in  pre-BML MURML.

AttributeTypeUseDescription
idStringoptionalmurml-description id (currently not used)
pointStrokeboolean (true, false)optional, defaults to falsetrue if the gesture has its stroke at a specific point (rather than a stroke phase)
scopeStringoptionalthe scope of the gesture

Child elements: (<keyframing> | <dynamic> | <static> | <symmetrical> | <parallel> | <sequence>)

TODO: implement <keyframing>. Keyframing is already supported when wrapped in dynamic. TODO: implement <sequence>

<keyframing>

AttributeTypeUseDescription
scope body | face optional
mode linear | spline | quaternion | raw optional
name string for animation name optional, defaults to KF_Anim_
easescale float from 1.0 to 50.0, reasonable values around 7.0 optional, default 1
easeturningpoint float from 0.0 to 1.0 optional, default 0.5
postpone_startframe double optional, default 0
dont_insert_startframe boolean optional, default false
notifyboolean ?
applymode SUPERPOSE|EXCLUSIVEoptional, default EXCLUSIVE
priority integer from -500 to +500 optional, default value 0
startTime float optional
endTime float optional

Child elements: <phase>+

TODO: check use of notify

TODO: implement SUPERPOSE applymode

<phase>

Used to group a bunch of keyframes into ONE continuous motion. Note: Each phase is timed locally, ie. its timing starts over from 0!

Child elements: (frame+)

<frame>

Encapsulates the description of a frame in a keyframe animation.

AttributeTypeUseDescription
ftimefloatrequiredfloat in secs

Child elements: (posture)

<posture>

Used to encapsulate a string that describes a configuration/posture either for the skeleton in terms of joint angles (the normal case), or for other body parts like the face (e.g. in terms of muscle values).

Child elements: #PCDATA

<dynamic>

AttributeTypeUseDescription
scopeString: typically neck, right_arm, left_armoptionalthe scope of the dynamic block
slotSlot: HandShape | HandLocation | PalmOrientation | ExtFingerOrientation | GazeDirection | Neckoptional

Child elements: ((dynamicElement+)|<keyframing>)

<static>

Specifies a single feature of a gesture that is to be held for a certain period of time.

AttributeTypeUseDescription
valueStringrequired
slotSlot: HandShape | HandLocation | PalmOrientation | ExtFingerOrientation | GazeDirection | Neckoptional
scopeString: typically neck, right_arm, left_armoptional
mode | pointTo optional, defaults to empty

TODO implement mode(?)

<symmetrical>

Enables a convenient way to specify a symmetrical gesture, namely, to define only one (the 'dominant') hand along with the specific symmetry constraints.

AttributeTypeUseDescription
dominant right_arm | left_arm required
symmetry SymM + any combination of up to three characters out of F|T|S (e.g: SymMS, SymMTF, SymMSFT) required

Child elements: (<parallel>|<sequence>|<static>|<dynamic>) TODO: implement sequence, static, dynamic

<parallel>

Can comprise any combination of constraints, among which it imposes simultaneity.

Child elements: ((<sequence>|<dynamic>|<static>)*)

TODO: implement use of sequence here TODO: DTD specifies this as (symmetrical)?,(sequence|static|dynamic)*. What does this mean?

<sequence>

Child elements: ((<symmetrical>|<dynamic>|<static>|<parallel>)*)

TODO: implement this

<dynamicElement>

AttributeTypeUseDescription
scopeString optional
typelinear | curve | chop optional

Child elements: (<value>+)

<value>

AttributeTypeUseDescription
typeString required
nameString required

Handshape Definition

handshape: basicHandShape|specificHandshape modification*
modification: (thumbShape)|(fingerSymbol)|(ThCpart [openingParameter])
fingerSymbol: fingerShape fingerParameter [openingParameter]

The default openingParameter is c (=1).

Some examples

BSfist (!ThExt)

Means use the BSFist handshape and overwrite the thumb with the ThExt handshape.

BSffinger (ThExt) (FBroll r) (FBroll l wo)

Means use the BSFist handshape, overwrite thumb with ThExt, and apply FBRoll on the ring finger and for 20% on the pinky.

Defining handshapes

The list of possible handShapes etc is provided in a hns xml file. For example:

<hns>
  <symbols>
    <symbol class="handShapes" name="BSfist" value=""/> 
    <symbol class="handShapes" name="BSflatc" value=""/>  
    ...
    <symbol class="basicHandShapes" name="BSfist" value=""/>
    <symbol class="basicHandShapes" name="BSflat" value=""/> 
    ...
    <symbol class="specificHandShapes" name="BSifinger"  value=""/> 
    <symbol class="specificHandShapes" name="BSimcfinger" value=""/>
    ...
    <symbol class="thumbShapes" name="ThExt" value=""/>
    <symbol class="thumbShapes" name="ThAcr" value=""/>
    ...  
    <symbol class="fingerShapes" name="FBstr" value=""/> 
    <symbol class="fingerShapes" name="FBangle" value=""/>
    ...
    <symbol class="fingerParameter" name="all" value=""/> 
    <symbol class="fingerParameter" name="l" value=""/> <!-- little finger (pinky) -->
    <symbol class="fingerParameter" name="r" value=""/> <!-- ring finger-->
    <symbol class="fingerParameter" name="m" value=""/> <!-- middle finger-->
    <symbol class="fingerParameter" name="p" value=""/> <!-- pointing (index) finger -->
    ...
    <symbol class="openingParameter" name="c" value=""/>   <!-- 1 -->
    <symbol class="openingParameter" name="no" value=""/>  <!-- 0.8 -->
    <symbol class="openingParameter" name="o" value=""/>   <!-- 0.5 -->
    <symbol class="openingParameter" name="wo" value=""/>  <!-- 0.2 -->
  </symbols> 
</hns>

HandLocation Definition

Specifying location

Hand location is specified using a triple which is either an x y z vector or 3 symbolic values (e.g. "LocShoulder LocCenterRight LocNorm"). The symbolic values define the z (in meter), phi (in degrees) and radial distance (rho, in meter) of the hand in a  cylindric coordinate system respectively. The coordinate system is relative to the basejoint specified in the hns xml above the hips (e.g. vl5).

http://upload.wikimedia.org/wikipedia/commons/0/0e/Coord_system_CY_1.svg

The symbolic values are define in the hns xml file. For example:

<hns>
  ...
  <basejoint sid="vl5"/>
  ...
  <symbols>
    <symbol class="handReferences" name="LocAboveHead" value="60"/>
    <symbol class="handReferences" name="LocHead" value="54"/>
    <symbol class="handReferences" name="LocForeHead" value="52"/>
    <symbol class="handReferences" name="LocEyes" value="50"/>
    <symbol class="handReferences" name="LocNose" value="48"/> 
    ...
    <symbol class="handLocators" name="LocCCenter" value="0"/>
    <symbol class="handLocators" name="LocCCenterLeft" value="15"/>
    <symbol class="handLocators" name="LocCenterLeft" value="25"/>
    <symbol class="handLocators" name="LocCenterRight" value="-25"/>
    ...
    <symbol class="handDistances" name="LocContact" value="10"/>
    <symbol class="handDistances" name="LocNear" value="15"/>
    <symbol class="handDistances" name="LocNorm" value="20"/>
    ...
  </symbols> 
</hns>

Specifying static HandLocation

Examples:

<static slot="HandLocation" value="LocShoulder LocCenterRight LocNorm" scope="left_arm"/>
<static slot="HandLocation" value="0.2 1 0.5"/>

Scope (optional) indicates which hand is used. Value is the location triple.

Specifying dynamic HandLocation

A dynamic HandLocation is specified within a dynamic block, using one or more dynamicElements. Each element defines a guiding stroke from the previous point to a new end position. The first dynamicElement also defines a start position for the guiding stroke. Depending on the type of guiding stroke, it might be required to specify other <value>s.

Specifying a linear guiding stroke

Example:

<dynamic slot="HandLocation" scope="left_arm">
  <dynamicElement type="linear" scope="left_arm">
    <value type="start" name="0.2 1.8 0.2"/>
    <value type="end" name="0.5 1.8 0.2"/>
  </dynamicElement>
  <dynamicElement type="linear" scope="left_arm">
    <value type="end" name="0.2 1.5 0.2"/>
  </dynamicElement>
</dynamic>

Specifying a curved guiding stroke

Example:

<dynamic slot="HandLocation" scope="left_arm">
  <dynamicElement type="curve">
    <value type="start"  name="LocShoulder LocCenterLeft LocFar"/>
    <value type="end"    name="LocShoulder LocCenterLeft LocNorm"/>
    <value type="normal" name="DirU"/>
    <value type="shape"  name="LeftC"/>
    <value type="extension" name="0.6"/>
  </dynamicElement>
  <dynamicElement type="curve">
    <value type="end"    name="LocShoulder LocCenterLeft LocFar"/>
    <value type="normal" name="DirU"/>
    <value type="shape"  name="LeftC"/>
    <value type="extension" name="0.6"/>
  </dynamicElement>
  <dynamicElement type="curve">
    <value type="end"    name="LocShoulder LocCenterLeft LocNorm"/>
    <value type="normal" name="DirU"/>
    <value type="shape"  name="LeftC"/>
    <value type="extension" name="0.6"/>
  </dynamicElement>
  <dynamicElement type="curve">
    <value type="end"    name="LocShoulder LocCenterLeft LocFar"/>
    <value type="normal" name="DirU"/>
    <value type="shape"  name="LeftC"/>
    <value type="extension" name="0.6"/>
  </dynamicElement>
</dynamic> 
ValueTypeUseDescription
normalDirectionrequiredTODO
shapeShapeSymbol (RightS, RightC, LeftC, LeftS)optional, defaults to RightSTODO
extensionfloatoptionalTODO
extentfloatoptional, defaults to 0.2TODO
roundnessfloatoptional, defaults to 0TODO
skewednessfloatoptional, defaults to 0TODO

TODO: is extension and extend the same?? Doublecheck original murml code+examples.

PalmOrientation Constraint

A Hand palm orientation constraint is defined either as an absolute direction of the palm in the world or as a local rotation of the wrist.

Absolute Constraint

Absolute constraints are defined using an x y z vector, or using 'Dir', followed by one or more of 'U' (up), 'D' (down), 'R' (right), 'L' (left), 'A' (forward), 'T' (backward).

Examples of absolute palm rotation constraints:

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <static slot="PalmOrientation" scope="left_arm" value="DirU"/> 
</murml-description>

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <dynamicElement>
    <value type="start" name="DirLU"/>
    <value type="end" name="DirU"/>
  </dynamicElement> 
</murml-description>

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <static slot="PalmOrientation" scope="left_arm" value="1 0 1"/> 
</murml-description>

Relative Constraint

Relative constraints are specified either as a single number denoting the local rotation of the left wrist (right wrist rotations are automatically mirrored) around the forward axis in degrees or as a symbolic string. The symbolic string maps to a rotation in degrees via the hns xml; by convention these rotations are of the form 'Palm' followed by one or more of 'U', 'D', 'R', 'L', 'A', 'T'.

Definition of some relative palm orientations in the hns xml.

<hns>
<symbols>
  ...
  <symbol class="palmOrientations" name="PalmL" value="180.0"/>
  <symbol class="palmOrientations" name="PalmLU" value="-135.0"/>
  <symbol class="palmOrientations" name="PalmLD" value="135"/>
  <symbol class="palmOrientations" name="PalmR" value="0.0"/>
  <symbol class="palmOrientations" name="PalmRU" value="-45.0"/>
  <symbol class="palmOrientations" name="PalmRD" value="45.0"/>
  <symbol class="palmOrientations" name="PalmU" value="-90.0"/>
  <symbol class="palmOrientations" name="PalmD" value="90"/>
</symbols>
</hns>

Examples of relative palm rotation constraints:

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <static slot="PalmOrientation" scope="left_arm" value="PalmU"/> 
</murml-description>

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <dynamicElement slot="PalmOrientation" scope="right_arm">
    <value type="start" name="PalmU"/>
    <value type="end" name="PalmD"/>
  </dynamicElement> 
</murml-description>

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <static slot="PalmOrientation" scope="left_arm" value="50"/> 
</murml-description>

ExtFingerOrientation Constraint

The ExtFingerOrientation constraint defines the absolute direction of the fingers in the same way as the absolute PalmOrientation constraint.

Examples of absolute ExtFingerOrientation constraints:

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <static slot="ExtFingerOrientation " scope="left_arm" value="DirTUULL"/> 
</murml-description>

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <dynamicElement slot="ExtFingerOrientation" scope="right_arm">
    <value type="start" name="DirLU"/>
    <value type="end" name="DirU"/>
  </dynamicElement> 
</murml-description>

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <static slot="ExtFingerOrientation" scope="left_arm" value="1 0 1"/> 
</murml-description>

Symmetry

Enables a convenient way to specify a symmetrical gesture, namely, to define only one (the 'dominant') hand along with the specific symmetry constraints.

<symmetrical dominant="right_arm | left_arm" symmetry=sym center=roll pitch yaw x y z>

Center contains 3 floats for rotation, followed by 3 floats for translation. sym is SymM + any combination of up to three characters out of F|T|S (e.g: SymMS, SymMTF, SymMSFT)

Using symmetry always results in mirror symmetry.

Sym: Mirror symmetry
SymMS Mirror symmetry, sag (=l/r).
SymMT Mirror symmetry, trans (=up/down)
SymMF Mirror symmetry, front (=forward/backward)

Keyframe Animation

Keyframe animations are packed within dynamic elements. The top level element is <keyframing>. The attributes mode, easescale, easeturningpoint and dont_insert_startframe are currently implemented. Keyframes may contain multiple phases [currently not implemented!]. Each phase is locally time, that is, starts at time 0. A phase has multiple animation <frame>s. If dont_insert_startframe is false (default), then an automatic transition from the current pose to the start of the keyframe is generated, in such a way that the first keyframe is attained at its specified ftime. Each frame contains a posture, which is written as

posture -> Humanoid posturespec*
postureSpec -> (jointid nrofdofvalues dofvalues)

For joint rotations this boils down to

(jointid 3 rolldegrees pitchdegrees yawdegrees)

And for morph targets to

(morphid 1 morphpercentage)

where morphpercentage may also be negative.

Example:

<murml-description xmlns="http://www.techfak.uni-bielefeld.de/ags/soa/murml">
  <dynamic>
    <keyframing>
        <phase>
        <frame ftime="1">
          <posture>Humanoid (r_shoulder 3 -33 -25 -25) (r_elbow 3 0 -91 0) (r_sternoclavicular 3 0 -17 -15) </posture>
        </frame>
        <frame ftime="1.4">
          <posture>Humanoid (r_shoulder 3 -33 -25 -25) (r_elbow 3 0 -110 0) (r_sternoclavicular 3 0 -17 -15) </posture>
        </frame>
        <frame ftime="1.8">
          <posture>Humanoid (r_shoulder 3 -33 -25 -25) (r_elbow 3 0 -91 0) (r_sternoclavicular 3 0 -17 -15) </posture>
        </frame>
        <frame ftime="2.2">
          <posture>Humanoid (r_shoulder 3 -33 -25 -25) (r_elbow 3 0 -110 0) (r_sternoclavicular 3 0 -17 -15) </posture>
        </frame>
        <frame ftime="2.6">
          <posture>Humanoid (r_shoulder 3 -33 -25 -25) (r_elbow 3 0 -91 0) (r_sternoclavicular 3 0 -17 -15) </posture>
        </frame>
        <frame ftime="2.8">
          <posture>Humanoid (r_shoulder 3 -33 -25 -25) (r_elbow 3 0 -91 0) (r_sternoclavicular 3 0 -17 -15) </posture>
        </frame>
        </phase>
    </keyframing>
  </dynamic>
</murml-description>