--- /dev/null
+% ---------------------------------------------------------------
+%
+% by Paolo.Ienne@di.epfl.ch
+%
+% ---------------------------------------------------------------
+%
+% no guarantee is given that the format corresponds perfectly to
+% IEEE 8.5" x 11" Proceedings, but most features should be ok.
+%
+% ---------------------------------------------------------------
+%
+% `ieee' from BibTeX standard bibliography style `abbrv'
+% version 0.99a for BibTeX versions 0.99a or later, LaTeX version 2.09.
+% Copyright (C) 1985, all rights reserved.
+% Copying of this file is authorized only if either
+% (1) you make absolutely no changes to your copy, including name, or
+% (2) if you do make changes, you name it something other than
+% btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst.
+% This restriction helps ensure that all standard styles are identical.
+% The file btxbst.doc has the documentation for this style.
+
+ENTRY
+ { address
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ howpublished
+ institution
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label }
+
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+
+STRINGS { s t }
+
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+ "\bibitem{" write$
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "{\em " swap$ * "}" * }
+ if$
+}
+
+INTEGERS { nameptr namesleft numnames }
+
+FUNCTION {format.names}
+{ 's :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr "{f.~}{vv~}{ll}{, jj}" format.name$ 't :=
+ nameptr #1 >
+ { namesleft #1 >
+ { ", " * t * }
+ { numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ t "others" =
+ { " et~al." * }
+ { " and " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {format.authors}
+{ author empty$
+ { "" }
+ { author format.names }
+ if$
+}
+
+FUNCTION {format.editors}
+{ editor empty$
+ { "" }
+ { editor format.names
+ editor num.names$ #1 >
+ { ", editors" * }
+ { ", editor" * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title empty$
+ { "" }
+ { title "t" change.case$ }
+ if$
+}
+
+FUNCTION {n.dashify}
+{ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {format.date}
+{ year empty$
+ { month empty$
+ { "" }
+ { "there's a month but no year in " cite$ * warning$
+ month
+ }
+ if$
+ }
+ { month empty$
+ 'year
+ { month " " * year * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.btitle}
+{ title emphasize
+}
+
+FUNCTION {tie.or.space.connect}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$ * *
+}
+
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { "volume" volume tie.or.space.connect
+ series empty$
+ 'skip$
+ { " of " * series emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+ { "number" }
+ { "Number" }
+ if$
+ number tie.or.space.connect
+ series empty$
+ { "there's a number but no series in " cite$ * warning$ }
+ { " in " * series * }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition empty$
+ { "" }
+ { output.state mid.sentence =
+ { edition "l" change.case$ " edition" * }
+ { edition "t" change.case$ " edition" * }
+ if$
+ }
+ if$
+}
+
+INTEGERS { multiresult }
+
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+
+FUNCTION {format.pages}
+{ pages empty$
+ { "" }
+ { pages multi.page.check
+ { "pages" pages n.dashify tie.or.space.connect }
+ { "page" pages tie.or.space.connect }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ number empty$
+ 'skip$
+ { "(" number * ")" * *
+ volume empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ }
+ if$
+ pages empty$
+ 'skip$
+ { duplicate$ empty$
+ { pop$ format.pages }
+ { ":" * pages n.dashify * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { "chapter" }
+ { type "l" change.case$ }
+ if$
+ chapter tie.or.space.connect
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.in.ed.booktitle}
+{ booktitle empty$
+ { "" }
+ { editor empty$
+ { "In " booktitle emphasize * }
+ { "In " format.editors * ", " * booktitle emphasize * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {empty.misc.check}
+{ author empty$ title empty$ howpublished empty$
+ month empty$ year empty$ note empty$
+ and and and and and
+ key empty$ not and
+ { "all relevant fields are empty in " cite$ * warning$ }
+ 'skip$
+ if$
+}
+
+FUNCTION {format.thesis.type}
+{ type empty$
+ 'skip$
+ { pop$
+ type "t" change.case$
+ }
+ if$
+}
+
+FUNCTION {format.tr.number}
+{ type empty$
+ { "Technical Report" }
+ 'type
+ if$
+ number empty$
+ { "t" change.case$ }
+ { number tie.or.space.connect }
+ if$
+}
+
+FUNCTION {format.article.crossref}
+{ key empty$
+ { journal empty$
+ { "need key or journal for " cite$ * " to crossref " * crossref *
+ warning$
+ ""
+ }
+ { "In {\em " journal * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.crossref.editor}
+{ editor #1 "{vv~}{ll}" format.name$
+ editor num.names$ duplicate$
+ #2 >
+ { pop$ " et~al." * }
+ { #2 <
+ 'skip$
+ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+ { " et~al." * }
+ { " and " * editor #2 "{vv~}{ll}" format.name$ * }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.book.crossref}
+{ volume empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ "In "
+ }
+ { "Volume" volume tie.or.space.connect
+ " of " *
+ }
+ if$
+ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { series empty$
+ { "need editor, key, or series for " cite$ * " to crossref " *
+ crossref * warning$
+ "" *
+ }
+ { "{\em " * series * "\/}" * }
+ if$
+ }
+ { key * }
+ if$
+ }
+ { format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.incoll.inproc.crossref}
+{ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { booktitle empty$
+ { "need editor, key, or booktitle for " cite$ * " to crossref " *
+ crossref * warning$
+ ""
+ }
+ { "In {\em " booktitle * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ }
+ { "In " format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { journal emphasize "journal" output.check
+ format.vol.num.pages output
+ format.date "year" output.check
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date "year" output.check
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.title "title" output.check
+ howpublished address new.block.checkb
+ howpublished output
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date "year" output.check
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ format.edition output
+ format.date "year" output.check
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ address empty$
+ { organization publisher new.sentence.checkb
+ organization output
+ publisher output
+ format.date "year" output.check
+ }
+ { address output.nonnull
+ format.date "year" output.check
+ new.sentence
+ organization output
+ publisher output
+ }
+ if$
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {conference} { inproceedings }
+
+FUNCTION {manual}
+{ output.bibitem
+ author empty$
+ { organization empty$
+ 'skip$
+ { organization output.nonnull
+ address output
+ }
+ if$
+ }
+ { format.authors output.nonnull }
+ if$
+ new.block
+ format.btitle "title" output.check
+ author empty$
+ { organization empty$
+ { address new.block.checka
+ address output
+ }
+ 'skip$
+ if$
+ }
+ { organization address new.block.checkb
+ organization output
+ address output
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ "Master's thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date "year" output.check
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ title howpublished new.block.checkb
+ format.title output
+ howpublished new.block.checka
+ howpublished output
+ format.date output
+ new.block
+ note output
+ fin.entry
+ empty.misc.check
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.btitle "title" output.check
+ new.block
+ "PhD thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date "year" output.check
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ editor empty$
+ { organization output }
+ { format.editors output.nonnull }
+ if$
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ address empty$
+ { editor empty$
+ { publisher new.sentence.checka }
+ { organization publisher new.sentence.checkb
+ organization output
+ }
+ if$
+ publisher output
+ format.date "year" output.check
+ }
+ { address output.nonnull
+ format.date "year" output.check
+ new.sentence
+ editor empty$
+ 'skip$
+ { organization output }
+ if$
+ publisher output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" output.check
+ address output
+ format.date "year" output.check
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ note "note" output.check
+ format.date output
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+
+MACRO {jan} {"Jan."}
+
+MACRO {feb} {"Feb."}
+
+MACRO {mar} {"Mar."}
+
+MACRO {apr} {"Apr."}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"Aug."}
+
+MACRO {sep} {"Sept."}
+
+MACRO {oct} {"Oct."}
+
+MACRO {nov} {"Nov."}
+
+MACRO {dec} {"Dec."}
+
+MACRO {acmcs} {"ACM Comput. Surv."}
+
+MACRO {acta} {"Acta Inf."}
+
+MACRO {cacm} {"Commun. ACM"}
+
+MACRO {ibmjrd} {"IBM J. Res. Dev."}
+
+MACRO {ibmsj} {"IBM Syst.~J."}
+
+MACRO {ieeese} {"IEEE Trans. Softw. Eng."}
+
+MACRO {ieeetc} {"IEEE Trans. Comput."}
+
+MACRO {ieeetcad}
+ {"IEEE Trans. Comput.-Aided Design Integrated Circuits"}
+
+MACRO {ipl} {"Inf. Process. Lett."}
+
+MACRO {jacm} {"J.~ACM"}
+
+MACRO {jcss} {"J.~Comput. Syst. Sci."}
+
+MACRO {scp} {"Sci. Comput. Programming"}
+
+MACRO {sicomp} {"SIAM J. Comput."}
+
+MACRO {tocs} {"ACM Trans. Comput. Syst."}
+
+MACRO {tods} {"ACM Trans. Database Syst."}
+
+MACRO {tog} {"ACM Trans. Gr."}
+
+MACRO {toms} {"ACM Trans. Math. Softw."}
+
+MACRO {toois} {"ACM Trans. Office Inf. Syst."}
+
+MACRO {toplas} {"ACM Trans. Prog. Lang. Syst."}
+
+MACRO {tcs} {"Theoretical Comput. Sci."}
+
+READ
+
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+
+INTEGERS { len }
+
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { nameptr #1 >
+ { " " * }
+ 'skip$
+ if$
+ s nameptr "{vv{ } }{ll{ }}{ f{ }}{ jj{ }}" format.name$ 't :=
+ nameptr numnames = t "others" = and
+ { "et al" * }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.organization.sort}
+{ author empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need author, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {editor.organization.sort}
+{ editor empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need editor, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+
+FUNCTION {presort}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.organization.sort
+ { type$ "manual" =
+ 'author.organization.sort
+ 'author.sort
+ if$
+ }
+ if$
+ }
+ if$
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+
+SORT
+
+STRINGS { longest.label }
+
+INTEGERS { number.label longest.label.width }
+
+FUNCTION {initialize.longest.label}
+{ "" 'longest.label :=
+ #1 'number.label :=
+ #0 'longest.label.width :=
+}
+
+FUNCTION {longest.label.pass}
+{ number.label int.to.str$ 'label :=
+ number.label #1 + 'number.label :=
+ label width$ longest.label.width >
+ { label 'longest.label :=
+ label width$ 'longest.label.width :=
+ }
+ 'skip$
+ if$
+}
+
+EXECUTE {initialize.longest.label}
+
+ITERATE {longest.label.pass}
+
+FUNCTION {begin.bib}
+{ preamble$ empty$
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+ "\begin{thebibliography}{" longest.label *
+ "}\setlength{\itemsep}{-1ex}\small" * write$ newline$
+}
+
+EXECUTE {begin.bib}
+
+EXECUTE {init.state.consts}
+
+ITERATE {call.type$}
+
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+
+EXECUTE {end.bib}
+
+% end of file ieee.bst
+% ---------------------------------------------------------------
--- /dev/null
+#LyX 1.2 created this file. For more info see http://www.lyx.org/
+\lyxformat 220
+\textclass report
+\begin_preamble
+\usepackage{fullpage}
+\renewenvironment{comment}%
+{\begin{quote}\textbf{Discussion}: \slshape}%
+{\end{quote}}
+\pagestyle{myheadings}
+\end_preamble
+\language american
+\inputencoding auto
+\fontscheme pslatex
+\graphics default
+\paperfontsize 10
+\spacing single
+\papersize letterpaper
+\paperpackage a4
+\use_geometry 0
+\use_amsmath 0
+\use_natbib 0
+\use_numerical_citations 0
+\paperorientation portrait
+\secnumdepth 2
+\tocdepth 2
+\paragraph_separation indent
+\defskip medskip
+\quotes_language english
+\quotes_times 2
+\papercolumns 1
+\papersides 2
+\paperpagestyle headings
+
+\layout Title
+
+The Portals 3.2 Message Passing Interface
+\newline
+ Revision 1.1
+\layout Author
+
+Ron Brightwell
+\begin_inset Foot
+collapsed true
+
+\layout Standard
+
+R.
+ Brightwell and R.
+ Riesen are with the Scalable Computing Systems Department, Sandia National
+ Laboratories, P.O.
+ Box 5800, Albuquerque, NM\SpecialChar ~
+\SpecialChar ~
+87111-1110, bright@cs.sandia.gov, rolf@cs.sandia.gov.
+\end_inset
+
+, Arthur B.
+ Maccabe
+\begin_inset Foot
+collapsed true
+
+\layout Standard
+
+A.
+ B.
+ Maccabe is with the Computer Science Department, University of New Mexico,
+ Albuquerque, NM\SpecialChar ~
+\SpecialChar ~
+87131-1386, maccabe@cs.unm.edu.
+\end_inset
+
+, Rolf Riesen and Trammell Hudson
+\layout Abstract
+
+This report presents a specification for the Portals 3.2 message passing
+ interface.
+ Portals 3.2 is intended to allow scalable, high-performance network communicatio
+n between nodes of a parallel computing system.
+ Specifically, it is designed to support a parallel computing platform composed
+ of clusters of commodity workstations connected by a commodity system area
+ network fabric.
+ In addition, Portals 3.2 is well suited to massively parallel processing
+ and embedded systems.
+ Portals 3.2 represents an adaption of the data movement layer developed
+ for massively parallel processing platforms, such as the 4500-node Intel
+ TeraFLOPS machine.
+
+\layout Standard
+
+
+\begin_inset ERT
+status Collapsed
+
+\layout Standard
+
+\backslash
+clearpage
+\backslash
+pagenumbering{roman}
+\backslash
+setcounter{page}{3}
+\end_inset
+
+
+\layout Standard
+
+
+\begin_inset LatexCommand \tableofcontents{}
+
+\end_inset
+
+
+\layout Standard
+
+
+\begin_inset ERT
+status Collapsed
+
+\layout Standard
+
+\backslash
+cleardoublepage
+\end_inset
+
+
+\layout Standard
+
+
+\begin_inset FloatList figure
+
+\end_inset
+
+
+\layout Standard
+
+
+\begin_inset ERT
+status Collapsed
+
+\layout Standard
+
+\backslash
+cleardoublepage
+\end_inset
+
+
+\layout Standard
+
+
+\begin_inset FloatList table
+
+\end_inset
+
+
+\layout Standard
+
+
+\begin_inset ERT
+status Collapsed
+
+\layout Standard
+
+\backslash
+cleardoublepage
+\end_inset
+
+
+\layout Chapter*
+
+Summary of Changes for Revision 1.1
+\layout Enumerate
+
+Updated version number to 3.2 throughout the document
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sub:PtlGetId}
+
+\end_inset
+
+: added
+\family typewriter
+PTL_SEGV
+\family default
+ to error list for
+\shape italic
+PtlGetId
+\shape default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:meattach}
+
+\end_inset
+
+: added
+\family typewriter
+PTL_ML_TOOLONG
+\family default
+ to error list for
+\shape italic
+PtlMEAttach
+\shape default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:meunlink}
+
+\end_inset
+
+: removed text referring to a list of associated memory descriptors.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:mdfree}
+
+\end_inset
+
+: added text to describe unlinking a free-floating memory descriptor.
+\layout Enumerate
+
+Table
+\begin_inset LatexCommand \ref{tab:types}
+
+\end_inset
+
+: added entry for
+\family typewriter
+ptl_seq_t
+\family default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:md-type}
+
+\end_inset
+
+:
+\begin_deeper
+\layout Enumerate
+
+added definition of
+\family typewriter
+max_offset
+\family default
+.
+\layout Enumerate
+
+added text to clarify
+\family typewriter
+PTL_MD_MANAGE_REMOTE
+\family default
+.
+\end_deeper
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:mdattach}
+
+\end_inset
+
+: modified text for
+\family typewriter
+unlink_op
+\family default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:niinit}
+
+\end_inset
+
+: added text to clarify multiple calls to
+\shape italic
+PtlNIInit
+\shape default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:mdattach}
+
+\end_inset
+
+: added text to clarify
+\family typewriter
+unlink_nofit
+\family default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:receiving}
+
+\end_inset
+
+: removed text indicating that an MD will reject a message if the associated
+ EQ is full.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:mdfree}
+
+\end_inset
+
+: added
+\family typewriter
+PTL_MD_INUSE
+\family default
+ error code and text to indicate that only MDs with no pending operations
+ can be unlinked.
+\layout Enumerate
+
+Table
+\begin_inset LatexCommand \ref{tab:retcodes}
+
+\end_inset
+
+: added
+\family typewriter
+PTL_MD_INUSE
+\family default
+ return code.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:event-type}
+
+\end_inset
+
+: added user id field, MD handle field, and NI specific failure field to
+ the
+\family typewriter
+ptl_event_t
+\family default
+ structure.
+\layout Enumerate
+
+Table
+\begin_inset LatexCommand \ref{tab:types}
+
+\end_inset
+
+: added
+\family typewriter
+ptl_ni_fail_t
+\family default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:event-type}
+
+\end_inset
+
+: added
+\family typewriter
+PTL_EVENT_UNLINK
+\family default
+ event type.
+\layout Enumerate
+
+Table
+\begin_inset LatexCommand \ref{tab:func}
+
+\end_inset
+
+: removed
+\shape slanted
+PtlTransId
+\shape default
+.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:meattach}
+
+\end_inset
+
+, Section
+\begin_inset LatexCommand \ref{sec:meinsert}
+
+\end_inset
+
+, Section
+\begin_inset LatexCommand \ref{sec:put}
+
+\end_inset
+
+: listed allowable constants with relevant fields.
+\layout Enumerate
+
+Table
+\begin_inset LatexCommand \ref{tab:func}
+
+\end_inset
+
+: added
+\shape italic
+PtlMEAttachAny
+\shape default
+ function.
+\layout Enumerate
+
+Table
+\begin_inset LatexCommand \ref{tab:retcodes}
+
+\end_inset
+
+: added
+\family typewriter
+PTL_PT_FULL
+\family default
+ return code for
+\shape italic
+PtlMEAttachAny
+\shape default
+.
+\layout Enumerate
+
+Table
+\begin_inset LatexCommand \ref{tab:oconsts}
+
+\end_inset
+
+: updated to reflect new event types.
+\layout Enumerate
+
+Section
+\begin_inset LatexCommand \ref{sec:id-type}
+
+\end_inset
+
+: added
+\family typewriter
+ptl_nid_t
+\family default
+,
+\family typewriter
+ptl_pid_t
+\family default
+, and
+\family typewriter
+ptl_uid_t
+\family default
+.
+\layout Chapter*
+
+Summary of Changes for Version 3.1
+\layout Section*
+
+Thread Issues
+\layout Standard
+
+The most significant change to the interface from version 3.0 to 3.1 involves
+ the clarification of how the interface interacts with multi-threaded applicatio
+ns.
+ We adopted a generic thread model in which processes define an address
+ space and threads share the address space.
+ Consideration of the API in the light of threads lead to several clarifications
+ throughout the document:
+\layout Enumerate
+
+Glossary:
+\begin_deeper
+\layout Enumerate
+
+added a definition for
+\emph on
+thread
+\emph default
+,
+\layout Enumerate
+
+reworded the definition for
+\emph on
+process
+\emph default
+.
+
+\end_deeper
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:apiover}
+
+\end_inset
+
+: added section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:threads}
+
+\end_inset
+
+ to describe the multi-threading model used by the Portals API.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:ptlinit}
+
+\end_inset
+
+:
+\emph on
+PtlInit
+\emph default
+ must be called at least once and may be called any number of times.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:ptlfini}
+
+\end_inset
+
+:
+\emph on
+PtlFini
+\emph default
+ should be called once as the process is terminating and not as each thread
+ terminates.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:pid}
+
+\end_inset
+
+: Portals does not define thread ids.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:ni}
+
+\end_inset
+
+: network interfaces are associated with processes, not threads.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:niinit}
+
+\end_inset
+
+:
+\emph on
+PtlNIInit
+\emph default
+ must be called at least once and may be called any number of times.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:eqget}
+
+\end_inset
+
+:
+\emph on
+PtlEQGet
+\emph default
+ returns
+\family typewriter
+PTL_EQ_EMPTY
+\family default
+ if a thread is blocked on
+\emph on
+PtlEQWait
+\emph default
+.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:eqwait}
+
+\end_inset
+
+: waiting threads are awakened in FIFO order.
+
+\layout Standard
+
+Two functions,
+\emph on
+PtlNIBarrier
+\emph default
+ and
+\emph on
+PtlEQCount
+\emph default
+ were removed from the API.
+
+\emph on
+PtlNIBarrier
+\emph default
+ was defined to block the calling process until all of the processes in
+ the application group had invoked
+\emph on
+PtlNIBarrier
+\emph default
+.
+ We now consider this functionality, along with the concept of groups (see
+ the discussion under
+\begin_inset Quotes eld
+\end_inset
+
+other changes
+\begin_inset Quotes erd
+\end_inset
+
+), to be part of the runtime system, not part of the Portals API.
+
+\emph on
+PtlEQCount
+\emph default
+ was defined to return the number of events in an event queue.
+ Because external operations may lead to new events being added and other
+ threads may remove events, the value returned by
+\emph on
+PtlEQCount
+\emph default
+ would have to be a hint about the number of events in the event queue.
+\layout Section*
+
+Handling small, unexpected messages
+\layout Standard
+
+Another set of changes relates to handling small unexpected messages in
+ MPI.
+ In designing version 3.0, we assumed that each unexpected message would
+ be placed in a unique memory descriptor.
+ To avoid the need to process a long list of memory descriptors, we moved
+ the memory descriptors out of the match list and hung them off of a single
+ match list entry.
+ In this way, large unexpected messages would only encounter a single
+\begin_inset Quotes eld
+\end_inset
+
+short message
+\begin_inset Quotes erd
+\end_inset
+
+ match list entry before encountering the
+\begin_inset Quotes eld
+\end_inset
+
+long message
+\begin_inset Quotes erd
+\end_inset
+
+ match list entry.
+ Experience with this strategy identified resource management problems with
+ this approach.
+ In particular, a long sequence of very short (or zero length) messages
+ could quickly exhaust the memory descriptors constructed for handling unexpecte
+d messages.
+ Our new strategy involves the use of several very large memory descriptors
+ for small unexpected messages.
+ Consecutive unexpected messages will be written into the first of these
+ memory descriptors until the memory descriptor fills up.
+ When the first of the
+\begin_inset Quotes eld
+\end_inset
+
+small memory
+\begin_inset Quotes erd
+\end_inset
+
+ descriptors fills up, it will be unlinked and subsequent short messages
+ will be written into the next
+\begin_inset Quotes eld
+\end_inset
+
+short message
+\begin_inset Quotes erd
+\end_inset
+
+ memory descriptor.
+ In this case, a
+\begin_inset Quotes eld
+\end_inset
+
+short message
+\begin_inset Quotes erd
+\end_inset
+
+ memory descriptor will be declared full when it does not have sufficient
+ space for the largest small unexpected message.
+\layout Standard
+
+This lead to two significant changes.
+ First, each match list entry now has a single memory descriptor rather
+ than a list of memory descriptors.
+ Second, in addition to exceeding the operation threshold, a memory descriptor
+ can be unlinked when the local offset exceeds a specified value.
+ These changes have lead to several changes in this document:
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{subsec:paddress}
+
+\end_inset
+
+:
+\begin_deeper
+\layout Enumerate
+
+removed references to the memory descriptor list,
+\layout Enumerate
+
+changed the portals address translation description to indicate that unlinking
+ a memory descriptor implies unlinking the associated match list entry--match
+ list entries can no longer be unlinked independently from the memory descriptor.
+
+\end_deeper
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:meattach}
+
+\end_inset
+
+:
+\begin_deeper
+\layout Enumerate
+
+removed unlink from argument list,
+\layout Enumerate
+
+removed description of
+\family typewriter
+ptl_unlink
+\family default
+ type,
+\layout Enumerate
+
+changed wording of the error condition when the Portal table index already
+ has an associated match list.
+
+\end_deeper
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:meinsert}
+
+\end_inset
+
+: removed unlink from argument list.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:md-type}
+
+\end_inset
+
+: added
+\family typewriter
+max_offset
+\family default
+.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:mdattach}
+
+\end_inset
+
+:
+\begin_deeper
+\layout Enumerate
+
+added description of
+\family typewriter
+ptl_unlink
+\family default
+ type,
+\layout Enumerate
+
+removed reference to memory descriptor lists,
+\layout Enumerate
+
+changed wording of the error condition when match list entry already has
+ an associated memory descriptor,
+\layout Enumerate
+
+changed the description of the
+\family typewriter
+unlink
+\family default
+ argument.
+
+\end_deeper
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:md}
+
+\end_inset
+
+: removed
+\family typewriter
+PtlMDInsert
+\family default
+ operation.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:mdbind}
+
+\end_inset
+
+: removed references to memory descriptor list.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:mdfree}
+
+\end_inset
+
+: removed reference to memory descriptor list.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:summary}
+
+\end_inset
+
+: removed references to PtlMDInsert.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:semantics}
+
+\end_inset
+
+: removed reference to memory descriptor list.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:exmpi}
+
+\end_inset
+
+: revised the MPI example to reflect the changes to the interface.
+
+\layout Standard
+
+Several changes have been made to improve the general documentation of the
+ interface.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:handle-type}
+
+\end_inset
+
+: documented the special value
+\family typewriter
+PTL_EQ_NONE
+\family default
+.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:id-type}
+
+\end_inset
+
+: documented the special value
+\family typewriter
+PTL_ID_ANY
+\family default
+.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:mdbind}
+
+\end_inset
+
+: documented the return value
+\family typewriter
+PTL_INV_EQ
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:mdupdate}
+
+\end_inset
+
+: clarified the description of the
+\emph on
+PtlMDUpdate
+\emph default
+ function.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:implvals}
+
+\end_inset
+
+: introduced a new section to document the implementation defined values.
+
+\layout Enumerate
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:summary}
+
+\end_inset
+
+: modified Table\SpecialChar ~
+
+\begin_inset LatexCommand \ref{tab:oconsts}
+
+\end_inset
+
+ to indicate where each constant is introduced and where it is used.
+
+\layout Section*
+
+Other changes
+\layout Subsection*
+
+Implementation defined limits (Section
+\begin_inset LatexCommand \ref{sec:niinit}
+
+\end_inset
+
+)
+\layout Standard
+
+The earlier version provided implementation defined limits for the maximum
+ number of match entries, the maximum number of memory descriptors, etc.
+ Rather than spanning the entire implementation, these limits are now associated
+ with individual network interfaces.
+\layout Subsection*
+
+Added User Ids (Section
+\begin_inset LatexCommand \ref{sec:uid}
+
+\end_inset
+
+)
+\layout Standard
+
+Group Ids had been used to simplify access control entries.
+ In particular, a process could allow access for all of the processes in
+ a group.
+ User Ids have been introduced to regain this functionality.
+ We use user ids to fill this role.
+\layout Subsection*
+
+Removed Group Ids and Rank Ids (Section
+\begin_inset LatexCommand \ref{sec:pid}
+
+\end_inset
+
+)
+\layout Standard
+
+The earlier version of Portals had two forms for addressing processes: <node
+ id, process id> and <group id, rank id>.
+ A process group was defined as the collection processes created during
+ application launch.
+ Each process in the group was given a unique rank id in the range 0 to
+
+\begin_inset Formula $n-1$
+\end_inset
+
+ where
+\begin_inset Formula $n$
+\end_inset
+
+ was the number of processes in the group.
+ We removed groups because they are better handled in the runtime system.
+\layout Subsection*
+
+Match lists (Section
+\begin_inset LatexCommand \ref{sec:meattach}
+
+\end_inset
+
+)
+\layout Standard
+
+It is no longer illegal to have an existing match entry when calling PtlMEAttach.
+ A position argument was added to the list of arguments supplied to
+\emph on
+PtlMEAttach
+\emph default
+ to specify whether the new match entry is prepended or appended to the
+ existing list.
+ If there is no existing match list, the position argument is ignored.
+\layout Subsection*
+
+Unlinking Memory Descriptors (Section
+\begin_inset LatexCommand \ref{sec:md}
+
+\end_inset
+
+)
+\layout Standard
+
+Previously, a memory descriptor could be unlinked if the offset exceeded
+ a threshold upon the completion of an operation.
+ In this version, the unlinking is delayed until there is a matching operation
+ which requires more memory than is currently available in the descriptor.
+ In addition to changes in section, this lead to a revision of Figure\SpecialChar ~
+
+\begin_inset LatexCommand \ref{fig:flow}
+
+\end_inset
+
+.
+\layout Subsection*
+
+Split Phase Operations and Events (Section
+\begin_inset LatexCommand \ref{sec:eq}
+
+\end_inset
+
+)
+\layout Standard
+
+Previously, there were five types of events:
+\family typewriter
+PTL_EVENT_PUT
+\family default
+,
+\family typewriter
+PTL_EVENT_GET
+\family default
+,
+\family typewriter
+PTL_EVENT_REPLY
+\family default
+,
+\family typewriter
+PTL_EVENT_SENT
+\family default
+, and
+\family typewriter
+PTL_EVENT_ACK.
+
+\family default
+The first four of these reflected the completion of potentially long operations.
+ We have introduced new event types to reflect the fact that long operations
+ have a distinct starting point and a distinct completion point.
+ Moreover, the completion may be successful or unsuccessful.
+\layout Standard
+
+In addition to providing a mechanism for reporting failure to higher levels
+ of software, this split provides an opportunity for for improved ordering
+ semantics.
+ Previously, if one process intiated two operations (e.g., two put operations)
+ on a remote process, these operations were guaranteed to complete in the
+ same order that they were initiated.
+ Now, we only guarantee that the initiation events are delivered in the
+ same order.
+ In particular, the operations do not need to complete in the order that
+ they were intiated.
+\layout Subsection*
+
+Well known proces ids (Section
+\begin_inset LatexCommand \ref{sec:niinit}
+
+\end_inset
+
+)
+\layout Standard
+
+To support the notion of
+\begin_inset Quotes eld
+\end_inset
+
+well known process ids,
+\begin_inset Quotes erd
+\end_inset
+
+ we added a process id argument to the arguments for PtlNIInit.
+\layout Chapter*
+
+Glossary
+\layout Description
+
+API Application Programming Interface.
+ A definition of the functions and semantics provided by library of functions.
+
+\layout Description
+
+Initiator A
+\emph on
+process
+\emph default
+ that initiates a message operation.
+
+\layout Description
+
+Message An application-defined unit of data that is exchanged between
+\emph on
+processes
+\emph default
+.
+
+\layout Description
+
+Message\SpecialChar ~
+Operation Either a put operation, which writes data, or a get operation,
+ which reads data.
+
+\layout Description
+
+Network A network provides point-to-point communication between
+\emph on
+nodes
+\emph default
+.
+ Internally, a network may provide multiple routes between endpoints (to
+ improve fault tolerance or to improve performance characteristics); however,
+ multiple paths will not be exposed outside of the network.
+
+\layout Description
+
+Node A node is an endpoint in a
+\emph on
+network
+\emph default
+.
+ Nodes provide processing capabilities and memory.
+ A node may provide multiple processors (an SMP node) or it may act as a
+
+\emph on
+gateway
+\emph default
+ between networks.
+
+\layout Description
+
+Process A context of execution.
+ A process defines a virtual memory (VM) context.
+ This context is not shared with other processes.
+ Several threads may share the VM context defined by a process.
+
+\layout Description
+
+Target A
+\emph on
+process
+\emph default
+ that is acted upon by a message operation.
+
+\layout Description
+
+Thread A context of execution that shares a VM context with other threads.
+
+\layout Standard
+
+
+\begin_inset ERT
+status Collapsed
+
+\layout Standard
+
+\backslash
+cleardoublepage
+\layout Standard
+
+\backslash
+setcounter{page}{1}
+\backslash
+pagenumbering{arabic}
+\end_inset
+
+
+\layout Chapter
+
+Introduction
+\begin_inset LatexCommand \label{sec:intro}
+
+\end_inset
+
+
+\layout Section
+
+Overview
+\layout Standard
+
+This document describes an application programming interface for message
+ passing between nodes in a system area network.
+ The goal of this interface is to improve the scalability and performance
+ of network communication by defining the functions and semantics of message
+ passing required for scaling a parallel computing system to ten thousand
+ nodes.
+ This goal is achieved by providing an interface that will allow a quality
+ implementation to take advantage of the inherently scalable design of Portals.
+\layout Standard
+
+This document is divided into several sections:
+\layout Description
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:intro}
+
+\end_inset
+
+---Introduction This section describes the purpose and scope of the Portals
+ API.
+
+\layout Description
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:apiover}
+
+\end_inset
+
+---An\SpecialChar ~
+Overview\SpecialChar ~
+of\SpecialChar ~
+the\SpecialChar ~
+Portals\SpecialChar ~
+3.1\SpecialChar ~
+API This section gives a brief overview of the
+ Portals API.
+ The goal is to introduce the key concepts and terminology used in the descripti
+on of the API.
+
+\layout Description
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:api}
+
+\end_inset
+
+---The\SpecialChar ~
+Portals\SpecialChar ~
+3.2\SpecialChar ~
+API This section describes the functions and semantics of
+ the Portals application programming interface.
+
+\layout Description
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:semantics}
+
+\end_inset
+
+--The\SpecialChar ~
+Semantics\SpecialChar ~
+of\SpecialChar ~
+Message\SpecialChar ~
+Transmission This section describes the semantics
+ of message transmission.
+ In particular, the information transmitted in each type of message and
+ the processing of incoming messages.
+
+\layout Description
+
+Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:examples}
+
+\end_inset
+
+---Examples This section presents several examples intended to illustrates
+ the use of the Portals API.
+
+\layout Section
+
+Purpose
+\layout Standard
+
+Existing message passing technologies available for commodity cluster networking
+ hardware do not meet the scalability goals required by the Cplant\SpecialChar ~
+
+\begin_inset LatexCommand \cite{Cplant}
+
+\end_inset
+
+ project at Sandia National Laboratories.
+ The goal of the Cplant project is to construct a commodity cluster that
+ can scale to the order of ten thousand nodes.
+ This number greatly exceeds the capacity for which existing message passing
+ technologies have been designed and implemented.
+\layout Standard
+
+In addition to the scalability requirements of the network, these technologies
+ must also be able to support a scalable implementation of the Message Passing
+ Interface (MPI)\SpecialChar ~
+
+\begin_inset LatexCommand \cite{MPIstandard}
+
+\end_inset
+
+ standard, which has become the
+\shape italic
+de facto
+\shape default
+ standard for parallel scientific computing.
+ While MPI does not impose any scalability limitations, existing message
+ passing technologies do not provide the functionality needed to allow implement
+ations of MPI to meet the scalability requirements of Cplant.
+\layout Standard
+
+The following are properties of a network architecture that do not impose
+ any inherent scalability limitations:
+\layout Itemize
+
+Connectionless - Many connection-oriented architectures, such as VIA\SpecialChar ~
+
+\begin_inset LatexCommand \cite{VIA}
+
+\end_inset
+
+ and TCP/IP sockets, have limitations on the number of peer connections
+ that can be established.
+
+\layout Itemize
+
+Network independence - Many communication systems depend on the host processor
+ to perform operations in order for messages in the network to be consumed.
+ Message consumption from the network should not be dependent on host processor
+ activity, such as the operating system scheduler or user-level thread scheduler.
+
+\layout Itemize
+
+User-level flow control - Many communication systems manage flow control
+ internally to avoid depleting resources, which can significantly impact
+ performance as the number of communicating processes increases.
+
+\layout Itemize
+
+OS Bypass - High performance network communication should not involve memory
+ copies into or out of a kernel-managed protocol stack.
+
+\layout Standard
+
+The following are properties of a network architecture that do not impose
+ scalability limitations for an implementation of MPI:
+\layout Itemize
+
+Receiver-managed - Sender-managed message passing implementations require
+ a persistent block of memory to be available for every process, requiring
+ memory resources to increase with job size and requiring user-level flow
+ control mechanisms to manage these resources.
+
+\layout Itemize
+
+User-level Bypass - While OS Bypass is necessary for high-performance, it
+ alone is not sufficient to support the Progress Rule of MPI asynchronous
+ operations.
+
+\layout Itemize
+
+Unexpected messages - Few communication systems have support for receiving
+ messages for which there is no prior notification.
+ Support for these types of messages is necessary to avoid flow control
+ and protocol overhead.
+
+\layout Section
+
+Background
+\layout Standard
+
+Portals was originally designed for and implemented on the nCube machine
+ as part of the SUNMOS (Sandia/UNM OS)\SpecialChar ~
+
+\begin_inset LatexCommand \cite{SUNMOS}
+
+\end_inset
+
+ and Puma\SpecialChar ~
+
+\begin_inset LatexCommand \cite{PumaOS}
+
+\end_inset
+
+ lightweight kernel development projects.
+ Portals went through two design phases, the latter of which is used on
+ the 4500-node Intel TeraFLOPS machine\SpecialChar ~
+
+\begin_inset LatexCommand \cite{TFLOPS}
+
+\end_inset
+
+.
+ Portals have been very successful in meeting the needs of such a large
+ machine, not only as a layer for a high-performance MPI implementation\SpecialChar ~
+
+\begin_inset LatexCommand \cite{PumaMPI}
+
+\end_inset
+
+, but also for implementing the scalable run-time environment and parallel
+ I/O capabilities of the machine.
+\layout Standard
+
+The second generation Portals implementation was designed to take full advantage
+ of the hardware architecture of large MPP machines.
+ However, efforts to implement this same design on commodity cluster technology
+ identified several limitations, due to the differences in network hardware
+ as well as to shortcomings in the design of Portals.
+\layout Section
+
+Scalability
+\layout Standard
+
+The primary goal in the design of Portals is scalability.
+ Portals are designed specifically for an implementation capable of supporting
+ a parallel job running on tens of thousands of nodes.
+ Performance is critical only in terms of scalability.
+ That is, the level of message passing performance is characterized by how
+ far it allows an application to scale and not by how it performs in micro-bench
+marks (e.g., a two node bandwidth or latency test).
+\layout Standard
+
+The Portals API is designed to allow for scalability, not to guarantee it.
+ Portals cannot overcome the shortcomings of a poorly designed application
+ program.
+ Applications that have inherent scalability limitations, either through
+ design or implementation, will not be transformed by Portals into scalable
+ applications.
+ Scalability must be addressed at all levels.
+ Portals do not inhibit scalability, but do not guarantee it either.
+\layout Standard
+
+To support scalability, the Portals interface maintains a minimal amount
+ of state.
+ Portals provide reliable, ordered delivery of messages between pairs of
+ processes.
+ They are connectionless: a process is not required to explicitly establish
+ a point-to-point connection with another process in order to communicate.
+ Moreover, all buffers used in the transmission of messages are maintained
+ in user space.
+ The target process determines how to respond to incoming messages, and
+ messages for which there are no buffers are discarded.
+\layout Section
+
+Communication Model
+\layout Standard
+
+Portals combine the characteristics of both one-side and two-sided communication.
+ They define a
+\begin_inset Quotes eld
+\end_inset
+
+matching put
+\begin_inset Quotes erd
+\end_inset
+
+ operation and a
+\begin_inset Quotes eld
+\end_inset
+
+matching get
+\begin_inset Quotes erd
+\end_inset
+
+ operation.
+ The destination of a put (or send) is not an explicit address; instead,
+ each message contains a set of match bits that allow the receiver to determine
+ where incoming messages should be placed.
+ This flexibility allows Portals to support both traditional one-sided operation
+s and two-sided send/receive operations.
+\layout Standard
+
+Portals allows the target to determine whether incoming messages are acceptable.
+ A target process can choose to accept message operations from any specific
+ process or can choose to ignore message operations from any specific process.
+\layout Section
+
+Zero Copy, OS Bypass and Application Bypass
+\layout Standard
+
+In traditional system architectures, network packets arrive at the network
+ interface card (NIC), are passed through one or more protocol layers in
+ the operating system, and eventually copied into the address space of the
+ application.
+ As network bandwidth began to approach memory copy rates, reduction of
+ memory copies became a critical concern.
+ This concern lead to the development of zero-copy message passing protocols
+ in which message copies are eliminated or pipelined to avoid the loss of
+ bandwidth.
+\layout Standard
+
+A typical zero-copy protocol has the NIC generate an interrupt for the CPU
+ when a message arrives from the network.
+ The interrupt handler then controls the transfer of the incoming message
+ into the address space of the appropriate application.
+ The interrupt latency, the time from the initiation of an interrupt until
+ the interrupt handler is running, is fairly significant.
+ To avoid this cost, some modern NICs have processors that can be programmed
+ to implement part of a message passing protocol.
+ Given a properly designed protocol, it is possible to program the NIC to
+ control the transfer of incoming messages, without needing to interrupt
+ the CPU.
+ Because this strategy does not need to involve the OS on every message
+ transfer, it is frequently called
+\begin_inset Quotes eld
+\end_inset
+
+OS Bypass.
+\begin_inset Quotes erd
+\end_inset
+
+ ST\SpecialChar ~
+
+\begin_inset LatexCommand \cite{ST}
+
+\end_inset
+
+, VIA\SpecialChar ~
+
+\begin_inset LatexCommand \cite{VIA}
+
+\end_inset
+
+, FM\SpecialChar ~
+
+\begin_inset LatexCommand \cite{FM2}
+
+\end_inset
+
+, GM\SpecialChar ~
+
+\begin_inset LatexCommand \cite{GM}
+
+\end_inset
+
+, and Portals are examples of OS Bypass protocols.
+\layout Standard
+
+Many protocols that support OS Bypass still require that the application
+ actively participate in the protocol to ensure progress.
+ As an example, the long message protocol of PM requires that the application
+ receive and reply to a request to put or get a long message.
+ This complicates the runtime environment, requiring a thread to process
+ incoming requests, and significantly increases the latency required to
+ initiate a long message protocol.
+ The Portals message passing protocol does not require activity on the part
+ of the application to ensure progress.
+ We use the term
+\begin_inset Quotes eld
+\end_inset
+
+Application Bypass
+\begin_inset Quotes erd
+\end_inset
+
+ to refer to this aspect of the Portals protocol.
+\layout Section
+
+Faults
+\layout Standard
+
+Given the number of components that we are dealing with and the fact that
+ we are interested in supporting applications that run for very long times,
+ failures are inevitable.
+ The Portals API recognizes that the underlying transport may not be able
+ to successfully complete an operation once it has been initiated.
+ This is reflected in the fact that the Portals API reports three types
+ of events: events indicating the initiation of an operation, events indicating
+ the successful completion of an operation, and events indicating the unsuccessf
+ul completion of an operation.
+ Every initiation event is eventually followed by a successful completion
+ event or an unsuccessful completion event.
+\layout Standard
+
+Between the time an operation is started and the time that the operation
+ completes (successfully or unsuccessfully), any memory associated with
+ the operation should be considered volatile.
+ That is, the memory may be changed in unpredictable ways while the operation
+ is progressing.
+ Once the operation completes, the memory associated with the operation
+ will not be subject to further modification (from this operation).
+ Notice that unsuccessful operations may alter memory in an essentially
+ unpredictable fashion.
+\layout Chapter
+
+An Overview of the Portals API
+\begin_inset LatexCommand \label{sec:apiover}
+
+\end_inset
+
+
+\layout Standard
+
+In this section, we give a conceptual overview of the Portals API.
+ The goal is to provide a context for understanding the detailed description
+ of the API presented in the next section.
+\layout Section
+
+Data Movement
+\begin_inset LatexCommand \label{sec:dmsemantics}
+
+\end_inset
+
+
+\layout Standard
+
+A Portal represents an opening in the address space of a process.
+ Other processes can use a Portal to read (get) or write (put) the memory
+ associated with the portal.
+ Every data movement operation involves two processes, the
+\series bold
+initiator
+\series default
+ and the
+\series bold
+target
+\series default
+.
+ The initiator is the process that initiates the data movement operation.
+ The target is the process that responds to the operation by either accepting
+ the data for a put operation, or replying with the data for a get operation.
+\layout Standard
+
+In this discussion, activities attributed to a process may refer to activities
+ that are actually performed by the process or
+\emph on
+on behalf of the process
+\emph default
+.
+ The inclusiveness of our terminology is important in the context of
+\emph on
+application bypass
+\emph default
+.
+ In particular, when we note that the target sends a reply in the case of
+ a get operation, it is possible that reply will be generated by another
+ component in the system, bypassing the application.
+\layout Standard
+
+Figures\SpecialChar ~
+
+\begin_inset LatexCommand \ref{fig:put}
+
+\end_inset
+
+ and
+\begin_inset LatexCommand \ref{fig:get}
+
+\end_inset
+
+ present graphical interpretations of the Portal data movement operations:
+ put and get.
+ In the case of a put operation, the initiator sends a put request message
+ containing the data to the target.
+ The target translates the Portal addressing information in the request
+ using its local Portal structures.
+ When the request has been processed, the target optionally sends an acknowledge
+ment message.
+\layout Standard
+
+
+\begin_inset Float figure
+placement htbp
+wide false
+collapsed false
+
+\layout Standard
+\align center
+
+\begin_inset Graphics FormatVersion 1
+ filename put.eps
+ display color
+ size_type 0
+ rotateOrigin center
+ lyxsize_type 1
+ lyxwidth 218pt
+ lyxheight 119pt
+\end_inset
+
+
+\layout Caption
+
+Portal Put (Send)
+\begin_inset LatexCommand \label{fig:put}
+
+\end_inset
+
+
+\end_inset
+
+
+\layout Standard
+
+In the case of a get operation, the initiator sends a get request to the
+ target.
+ As with the put operation, the target translates the Portal addressing
+ information in the request using its local Portal structures.
+ Once it has translated the Portal addressing information, the target sends
+ a reply that includes the requested data.
+\layout Standard
+
+
+\begin_inset Float figure
+placement htbp
+wide false
+collapsed false
+
+\layout Standard
+\align center
+
+\begin_inset Graphics FormatVersion 1
+ filename get.eps
+ display color
+ size_type 0
+ rotateOrigin center
+ lyxsize_type 1
+ lyxwidth 218pt
+ lyxheight 119pt
+\end_inset
+
+
+\layout Caption
+
+Portal Get
+\begin_inset LatexCommand \label{fig:get}
+
+\end_inset
+
+
+\end_inset
+
+
+\layout Standard
+
+We should note that Portal address translations are only performed on nodes
+ that respond to operations initiated by other nodes.
+ Acknowledgements and replies to get operations bypass the portals address
+ translation structures.
+\layout Section
+
+Portal Addressing
+\begin_inset LatexCommand \label{subsec:paddress}
+
+\end_inset
+
+
+\layout Standard
+
+One-sided data movement models (e.g., shmem\SpecialChar ~
+
+\begin_inset LatexCommand \cite{CraySHMEM}
+
+\end_inset
+
+, ST\SpecialChar ~
+
+\begin_inset LatexCommand \cite{ST}
+
+\end_inset
+
+, MPI-2\SpecialChar ~
+
+\begin_inset LatexCommand \cite{MPI2}
+
+\end_inset
+
+) typically use a triple to address memory on a remote node.
+ This triple consists of a process id, memory buffer id, and offset.
+ The process id identifies the target process, the memory buffer id specifies
+ the region of memory to be used for the operation, and the offset specifies
+ an offset within the memory buffer.
+\layout Standard
+
+In addition to the standard address components (process id, memory buffer
+ id, and offset), a Portal address includes a set of match bits.
+ This addressing model is appropriate for supporting one-sided operations
+ as well as traditional two-sided message passing operations.
+ Specifically, the Portals API provides the flexibility needed for an efficient
+ implementation of MPI-1, which defines two-sided operations with one-sided
+ completion semantics.
+\layout Standard
+
+Figure\SpecialChar ~
+
+\begin_inset LatexCommand \ref{fig:portals}
+
+\end_inset
+
+ presents a graphical representation of the structures used by a target
+ in the interpretation of a Portal address.
+ The process id is used to route the message to the appropriate node and
+ is not reflected in this diagram.
+ The memory buffer id, called the
+\series bold
+portal id
+\series default
+, is used as an index into the Portal table.
+ Each element of the Portal table identifies a match list.
+ Each element of the match list specifies two bit patterns: a set of
+\begin_inset Quotes eld
+\end_inset
+
+don't care
+\begin_inset Quotes erd
+\end_inset
+
+ bits, and a set of
+\begin_inset Quotes eld
+\end_inset
+
+must match
+\begin_inset Quotes erd
+\end_inset
+
+ bits.
+ In addition to the two sets of match bits, each match list element has
+ at most one memory descriptor.
+ Each memory descriptor identifies a memory region and an optional event
+ queue.
+ The memory region specifies the memory to be used in the operation and
+ the event queue is used to record information about these operations.
+\layout Standard
+
+
+\begin_inset Float figure
+placement htbp
+wide false
+collapsed false
+
+\layout Standard
+\align center
+
+\begin_inset Graphics FormatVersion 1
+ filename portals.eps
+ display color
+ size_type 0
+ rotateOrigin center
+ lyxsize_type 1
+ lyxwidth 305pt
+ lyxheight 106pt
+\end_inset
+
+
+\layout Caption
+
+Portal Addressing Structures
+\begin_inset LatexCommand \label{fig:portals}
+
+\end_inset
+
+
+\end_inset
+
+
+\layout Standard
+
+Figure\SpecialChar ~
+
+\begin_inset LatexCommand \ref{fig:flow}
+
+\end_inset
+
+ illustrates the steps involved in translating a Portal address, starting
+ from the first element in a match list.
+ If the match criteria specified in the match list entry are met and the
+ memory descriptor list accepts the operation
+\begin_inset Foot
+collapsed true
+
+\layout Standard
+
+Memory descriptors can reject operations because a threshold has been exceeded
+ or because the memory region does not have sufficient space, see Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:md}
+
+\end_inset
+
+
+\end_inset
+
+, the operation (put or get) is performed using the memory region specified
+ in the memory descriptor.
+ If the memory descriptor specifies that it is to be unlinked when a threshold
+ has been exceeded, the match list entry is removed from the match list
+ and the resources associated with the memory descriptor and match list
+ entry are reclaimed.
+ Finally, if there is an event queue specified in the memory descriptor,
+ the operation is logged in the event queue.
+\layout Standard
+
+
+\begin_inset Float figure
+placement htbp
+wide false
+collapsed false
+
+\layout Standard
+\align center
+
+\begin_inset Graphics FormatVersion 1
+ filename flow_new.eps
+ display color
+ size_type 0
+ rotateOrigin center
+ lyxsize_type 1
+ lyxwidth 447pt
+ lyxheight 282pt
+\end_inset
+
+
+\layout Caption
+
+Portals Address Translation
+\begin_inset LatexCommand \label{fig:flow}
+
+\end_inset
+
+
+\end_inset
+
+
+\layout Standard
+
+If the match criteria specified in the match list entry are not met, or
+ there is no memory descriptor associated with the match list entry, or
+ the memory descriptor associated with the match list entry rejects the
+ operation, the address translation continues with the next match list entry.
+ If the end of the match list has been reached, the address translation
+ is aborted and the incoming requested is discarded.
+\layout Section
+
+Access Control
+\layout Standard
+
+A process can control access to its portals using an access control list.
+ Each entry in the access control list specifies a process id and a Portal
+ table index.
+ The access control list is actually an array of entries.
+ Each incoming request includes an index into the access control list (i.e.,
+ a
+\begin_inset Quotes eld
+\end_inset
+
+cookie
+\begin_inset Quotes erd
+\end_inset
+
+ or hint).
+ If the id of the process issuing the request doesn't match the id specified
+ in the access control list entry or the Portal table index specified in
+ the request doesn't match the Portal table index specified in the access
+ control list entry, the request is rejected.
+ Process identifiers and Portal table indexes may include wild card values
+ to increase the flexibility of this mechanism.
+
+\layout Standard
+
+Two aspects of this design merit further discussion.
+ First, the model assumes that the information in a message header, the
+ sender's id in particular, is trustworthy.
+ In most contexts, we assume that the entity that constructs the header
+ is trustworthy; however, using cryptographic techniques, we could easily
+ devise a protocol that would ensure the authenticity of the sender.
+\layout Standard
+
+Second, because the access check is performed by the receiver, it is possible
+ that a malicious process will generate thousands of messages that will
+ be denied by the receiver.
+ This could saturate the network and/or the receiver, resulting in a
+\emph on
+denial of service
+\emph default
+ attack.
+ Moving the check to the sender using capabilities, would remove the potential
+ for this form of attack.
+ However, the solution introduces the complexities of capability management
+ (exchange of capabilities, revocation, protections, etc).
+\layout Section
+
+Multi-threaded Applications
+\begin_inset LatexCommand \label{sec:threads}
+
+\end_inset
+
+
+\layout Standard
+
+The Portals API supports a generic view of multi-threaded applications.
+ From the perspective of the Portals API, an application program is defined
+ by a set of processes.
+ Each process defines a unique address space.
+ The Portals API defines access to this address space from other processes
+ (using portals addressing and the data movement operations).
+ A process may have one or more
+\emph on
+threads
+\emph default
+ executing in its address space.
+
+\layout Standard
+
+With the exception of
+\emph on
+PtlEQWait
+\emph default
+ every function in the Portals API is non-blocking and atomic with respect
+ to both other threads and external operations that result from data movement
+ operations.
+ While individual operations are atomic, sequences of these operations may
+ be interleaved between different threads and with external operations.
+ The Portals API does not provide any mechanisms to control this interleaving.
+ It is expected that these mechanisms will be provided by the API used to
+ create threads.
+\layout Chapter
+
+The Portals API
+\begin_inset LatexCommand \label{sec:api}
+
+\end_inset
+
+
+\layout Section
+
+Naming Conventions
+\begin_inset LatexCommand \label{sec:conv}
+
+\end_inset
+
+
+\layout Standard
+
+The Portals API defines two types of entities: functions and types.
+ Function always start with
+\emph on
+Ptl
+\emph default
+ and use mixed upper and lower case.
+ When used in the body of this report, function names appear in italic face,
+ e.g.,
+\emph on
+PtlInit
+\emph default
+.
+ The functions associated with an object type will have names that start
+ with
+\emph on
+Ptl
+\emph default
+, followed by the two letter object type code shown in Table\SpecialChar ~
+
+\begin_inset LatexCommand \ref{tab:objcodes}
+
+\end_inset
+
+.
+ As an example, the function
+\emph on
+PtlEQAlloc
+\emph default
+ allocates resources for an event queue.
+\layout Standard
+
+
+\begin_inset Float table
+placement htbp
+wide false
+collapsed false
+
+\layout Caption
+
+Object Type Codes
+\begin_inset LatexCommand \label{tab:objcodes}
+
+\end_inset
+
+
+\begin_inset ERT
+status Collapsed
+
+\layout Standard
+
+\backslash
+medskip
+\newline
+
+\end_inset
+
+
+\layout Standard
+\align center
+
+\size small
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="5" columns="3">
+<features firstHeadEmpty="true">
+<column alignment="left" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="0pt">
+<row bottomline="true">
+<cell alignment="left" valignment="top" bottomline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\emph on
+xx
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" bottomline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ Name
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" bottomline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ Section
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+EQ
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ Event Queue
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\begin_inset LatexCommand \ref{sec:eq}
+
+\end_inset
+
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ MD
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ Memory Descriptor
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\begin_inset LatexCommand \ref{sec:md}
+
+\end_inset
+
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ ME
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ Match list Entry
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\begin_inset LatexCommand \ref{sec:me}
+
+\end_inset
+
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ NI
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+ Network Interface
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\begin_inset LatexCommand \ref{sec:ni}
+
+\end_inset
+
+
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\end_inset
+
+
+\layout Standard
+
+Type names use lower case with underscores to separate words.
+ Each type name starts with
+\family typewriter
+ptl
+\family default
+_ and ends with
+\family typewriter
+_t
+\family default
+.
+ When used in the body of this report, type names appear in a fixed font,
+ e.g.,
+\family typewriter
+ptl_match_bits_t
+\family default
+.
+\layout Standard
+
+Names for constants use upper case with underscores to separate words.
+ Each constant name starts with
+\family typewriter
+PTL_
+\family default
+.
+ When used in the body of this report, type names appear in a fixed font,
+ e.g.,
+\family typewriter
+PTL_OK
+\family default
+.
+\layout Section
+
+Base Types
+\layout Standard
+
+The Portals API defines a variety of base types.
+ These types represent a simple renaming of the base types provided by the
+ C programming language.
+ In most cases these new type names have been introduced to improve type
+ safety and to avoid issues arising from differences in representation sizes
+ (e.g., 16-bit or 32-bit integers).
+\layout Subsection
+
+Sizes
+\begin_inset LatexCommand \label{sec:size-t}
+
+\end_inset
+
+
+\layout Standard
+
+The type
+\family typewriter
+ptl_size_t
+\family default
+ is an unsigned 64-bit integral type used for representing sizes.
+\layout Subsection
+
+Handles
+\begin_inset LatexCommand \label{sec:handle-type}
+
+\end_inset
+
+
+\layout Standard
+
+Objects maintained by the API are accessed through handles.
+ Handle types have names of the form
+\family typewriter
+ptl_handle_
+\emph on
+xx
+\emph default
+_t
+\family default
+, where
+\emph on
+xx
+\emph default
+ is one of the two letter object type codes shown in Table\SpecialChar ~
+
+\begin_inset LatexCommand \ref{tab:objcodes}
+
+\end_inset
+
+.
+ For example, the type
+\family typewriter
+ptl_handle_ni_t
+\family default
+ is used for network interface handles.
+\layout Standard
+
+Each type of object is given a unique handle type to enhance type checking.
+ The type,
+\family typewriter
+ptl_handle_any_t
+\family default
+, can be used when a generic handle is needed.
+ Every handle value can be converted into a value of type
+\family typewriter
+ptl_handle_any_t
+\family default
+ without loss of information.
+\layout Standard
+
+Handles are not simple values.
+ Every portals object is associated with a specific network interface and
+ an identifier for this interface (along with an object identifier) is part
+ of the handle for the object.
+\layout Standard
+
+The special value
+\family typewriter
+PTL_EQ_NONE
+\family default
+, of type
+\family typewriter
+ptl_handle_eq_t
+\family default
+, is used to indicate the absence of an event queue.
+ See sections
+\begin_inset LatexCommand \ref{sec:mdfree}
+
+\end_inset
+
+ and\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:mdupdate}
+
+\end_inset
+
+ for uses of this value.
+\layout Subsection
+
+Indexes
+\begin_inset LatexCommand \label{sec:index-type}
+
+\end_inset
+
+
+\layout Standard
+
+The types
+\family typewriter
+ptl_pt_index_t
+\family default
+ and
+\family typewriter
+ptl_ac_index_t
+\family default
+ are integral types used for representing Portal table indexes and access
+ control tables indexes, respectively.
+ See section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:niinit}
+
+\end_inset
+
+ for limits on values of these types.
+\layout Subsection
+
+Match Bits
+\begin_inset LatexCommand \label{sec:mb-type}
+
+\end_inset
+
+
+\layout Standard
+
+The type
+\family typewriter
+ptl_match_bits_t
+\family default
+ is capable of holding unsigned 64-bit integer values.
+\layout Subsection
+
+Network Interfaces
+\begin_inset LatexCommand \label{sec:ni-type}
+
+\end_inset
+
+
+\layout Standard
+
+The type
+\family typewriter
+ptl_interface_t
+\family default
+ is an integral type used for identifying different network interfaces.
+ Users will need to consult the local documentation to determine appropriate
+ values for the interfaces available.
+ The special value
+\family typewriter
+PTL_IFACE_DEFAULT
+\family default
+ identifies the default interface.
+\layout Subsection
+
+Identifiers
+\begin_inset LatexCommand \label{sec:id-type}
+
+\end_inset
+
+
+\layout Standard
+
+The type
+\family typewriter
+ptl_nid_t
+\family default
+ is an integral type used for representing node ids
+\family typewriter
+, ptl_pid_t
+\family default
+ is an integral type for representing process ids, and
+\family typewriter
+ptl_uid_t
+\family default
+is an integral type for representing user ids.
+\layout Standard
+
+The special values
+\family typewriter
+PTL_PID_ANY
+\family default
+ matches any process identifier, PTL_NID_ANY matches any node identifier,
+ and
+\family typewriter
+PTL_UID_ANY
+\family default
+ matches any user identifier.
+ See sections
+\begin_inset LatexCommand \ref{sec:meattach}
+
+\end_inset
+
+ and\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:acentry}
+
+\end_inset
+
+ for uses of these values.
+\layout Subsection
+
+Status Registers
+\begin_inset LatexCommand \label{sec:stat-type}
+
+\end_inset
+
+
+\layout Standard
+
+Each network interface maintains an array of status registers that can be
+ accessed using the
+\family typewriter
+PtlNIStatus
+\family default
+ function (see Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:nistatus}
+
+\end_inset
+
+).
+ The type
+\family typewriter
+ptl_sr_index_t
+\family default
+ defines the types of indexes that can be used to access the status registers.
+ The only index defined for all implementations is
+\family typewriter
+PTL_SR_DROP_COUNT
+\family default
+ which identifies the status register that counts the dropped requests for
+ the interface.
+ Other indexes (and registers) may be defined by the implementation.
+\layout Standard
+
+The type
+\family typewriter
+ptl_sr_value_t
+\family default
+ defines the types of values held in status registers.
+ This is a signed integer type.
+ The size is implementation dependent, but must be at least 32 bits.
+\layout Section
+
+Initialization and Cleanup
+\begin_inset LatexCommand \label{sec:init}
+
+\end_inset
+
+
+\layout Standard
+
+The Portals API includes a function,
+\emph on
+PtlInit
+\emph default
+, to initialize the library and a function,
+\emph on
+PtlFini
+\emph default
+, to cleanup after the application is done using the library.
+\layout Subsection
+
+PtlInit
+\begin_inset LatexCommand \label{sec:ptlinit}
+
+\end_inset
+
+
+\layout LyX-Code
+
+int PtlInit( int *max_interfaces );
+\layout Standard
+\noindent
+The
+\emph on
+PtlInit
+\emph default
+ function initializes the Portals library.
+ PtlInit must be called at least once by a process before any thread makes
+ a Portals function call, but may be safely called more than once.
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_FAIL Indicates an error during initialization.
+
+\layout Description
+
+PTL_SEGV Indicates that
+\family typewriter
+max_interfaces
+\family default
+ is not a legal address.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="1" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="5in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+max_interfaces
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, this location will hold the maximum number of interfaces
+ that can be initialized.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Subsection
+
+PtlFini
+\begin_inset LatexCommand \label{sec:ptlfini}
+
+\end_inset
+
+
+\layout LyX-Code
+
+void PtlFini( void );
+\layout Standard
+\noindent
+The
+\emph on
+PtlFini
+\emph default
+ function cleans up after the Portals library is no longer needed by a process.
+ After this function is called, calls to any of the functions defined by
+ the Portal API or use of the structures set up by the Portals API will
+ result in undefined behavior.
+ This function should be called once and only once during termination by
+ a process.
+ Typically, this function will be called in the exit sequence of a process.
+ Individual threads should not call PtlFini when they terminate.
+\layout Section
+
+Network Interfaces
+\begin_inset LatexCommand \label{sec:ni}
+
+\end_inset
+
+
+\layout Standard
+
+The Portals API supports the use of multiple network interfaces.
+ However, each interface is treated as an independent entity.
+ Combining interfaces (e.g.,
+\begin_inset Quotes eld
+\end_inset
+
+bonding
+\begin_inset Quotes erd
+\end_inset
+
+ to create a higher bandwidth connection) must be implemented by the application
+ or embedded in the underlying network.
+ Interfaces are treated as independent entities to make it easier to cache
+ information on individual network interface cards.
+\layout Standard
+
+Once initialized, each interface provides a Portal table, an access control
+ table, and a collection of status registers.
+ See Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:me}
+
+\end_inset
+
+ for a discussion of updating Portal table entries using the
+\emph on
+PtlMEAttach
+\emph default
+ function.
+ See Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:ac}
+
+\end_inset
+
+ for a discussion of the initialization and updating of entries in the access
+ control table.
+ See Section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:nistatus}
+
+\end_inset
+
+ for a discussion of the
+\emph on
+PtlNIStatus
+\emph default
+ function which can be used to determine the value of a status register.
+\layout Standard
+
+Every other type of Portal object (e.g., memory descriptor, event queue, or
+ match list entry) is associated with a specific network interface.
+ The association to a network interface is established when the object is
+ created and is encoded in the handle for the object.
+\layout Standard
+
+Each network interface is initialized and shutdown independently.
+ The initialization routine,
+\emph on
+PtlNIInit
+\emph default
+, returns a handle for an interface object which is used in all subsequent
+ Portal operations.
+ The
+\emph on
+PtlNIFini
+\emph default
+ function is used to shutdown an interface and release any resources that
+ are associated with the interface.
+ Network interface handles are associated with processes, not threads.
+ All threads in a process share all of the network interface handles.
+\layout Standard
+
+The Portals API also defines the
+\emph on
+PtlNIStatus
+\emph default
+ function to query the status registers for a network interface, the
+\emph on
+PtlNIDist
+\emph default
+ function to determine the
+\begin_inset Quotes eld
+\end_inset
+
+distance
+\begin_inset Quotes erd
+\end_inset
+
+ to another process, and the
+\emph on
+PtlNIHandle
+\emph default
+ function to determine the network interface that an object is associated
+ with.
+\layout Subsection
+
+PtlNIInit
+\begin_inset LatexCommand \label{sec:niinit}
+
+\end_inset
+
+
+\layout LyX-Code
+
+typedef struct {
+\newline
+ int max_match_entries;
+\newline
+ int max_mem_descriptors;
+\newline
+ int max_event_queues;
+\newline
+ ptl_ac_index_t max_atable_index;
+\newline
+ ptl_pt_index_t max_ptable_index;
+\newline
+} ptl_ni_limits_t;
+\newline
+
+\newline
+int PtlNIInit( ptl_interface_t interface
+\newline
+ ptl_pid_t pid,
+\newline
+ ptl_ni_limits_t* desired,
+\newline
+ ptl_ni_limits_t* actual,
+\newline
+ ptl_handle_ni_t* handle );
+\layout Standard
+
+Values of type
+\family typewriter
+ptl_ni_limits_t
+\family default
+ include the following members:
+\layout Description
+
+max_match_entries Maximum number of match entries that can be allocated
+ at any one time.
+\layout Description
+
+max_mem_descriptors Maximum number of memory descriptors that can be allocated
+ at any one time.
+\layout Description
+
+max_event_queues Maximum number of event queues that can be allocated at
+ any one time.
+\layout Description
+
+max_atable_index Largest access control table index for this interface,
+ valid indexes range from zero to
+\family typewriter
+max_atable_index
+\family default
+, inclusive.
+\layout Description
+
+max_ptable_index Largest Portal table index for this interface, valid indexes
+ range from zero to
+\family typewriter
+max_ptable_index
+\family default
+, inclusive.
+\layout Standard
+\noindent
+The
+\emph on
+PtlNIInit
+\emph default
+ function is used to initialized the Portals API for a network interface.
+ This function must be called at least once by each process before any other
+ operations that apply to the interface by any process or thread.
+ For subsequent calls to
+\shape italic
+PtlNIInit
+\shape default
+ from within the same process (either by different threads or the same thread),
+ the desired limits will be ignored and the call will return the existing
+ NI handle.
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_INIT_DUP Indicates a duplicate initialization of
+\family typewriter
+interface
+\family default
+.
+
+\layout Description
+
+PTL_INIT_INV Indicates that
+\family typewriter
+interface
+\family default
+ is not a valid network interface.
+
+\layout Description
+
+PTL_NOSPACE Indicates that there is insufficient memory to initialize the
+ interface.
+
+\layout Description
+
+PTL_INV_PROC Indicates that
+\family typewriter
+pid
+\family default
+ is not a valid process id.
+\layout Description
+
+PTL_SEGV Indicates that
+\family typewriter
+actual
+\family default
+or
+\family typewriter
+ handle
+\family default
+ is not a legal address.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="5" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="4.7in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+interface
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+Identifies the network interface to be initialized.
+ (See section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:ni-type}
+
+\end_inset
+
+ for a discussion of values used to identify network interfaces.)
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+pid
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+Identifies the desired process id (for well known process ids).
+ The value
+\family typewriter
+PTL_PID_ANY
+\family default
+ may be used to have the process id assigned by the underlying library.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+desired
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+If non-NULL, points to a structure that holds the desired limits.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+actual
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, the location pointed to by actual will hold the actual
+ limits.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+handle
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, this location will hold a handle for the interface.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Comment
+
+The use of desired is implementation dependent.
+ In particular, an implementation may choose to ignore this argument.
+\layout Subsection
+
+PtlNIFini
+\begin_inset LatexCommand \label{sec:nifini}
+
+\end_inset
+
+
+\layout LyX-Code
+
+int PtlNIFini( ptl_handle_ni_t interface );
+\layout Standard
+\noindent
+The
+\emph on
+PtlNIFini
+\emph default
+ function is used to release the resources allocated for a network interface.
+ Once the
+\emph on
+PtlNIFini
+\emph default
+ operation has been started, the results of pending API operations (e.g.,
+ operations initiated by another thread) for this interface are undefined.
+ Similarly, the effects of incoming operations (puts and gets) or return
+ values (acknowledgements and replies) for this interface are undefined.
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_INV_NI Indicates that
+\family typewriter
+interface
+\family default
+ is not a valid network interface handle.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="1" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+interface
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+A handle for the interface to shutdown.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Subsection
+
+PtlNIStatus
+\begin_inset LatexCommand \label{sec:nistatus}
+
+\end_inset
+
+
+\layout LyX-Code
+
+int PtlNIStatus( ptl_handle_ni_t interface,
+\newline
+ ptl_sr_index_t status_register,
+\newline
+ ptl_sr_value_t* status );
+\layout Standard
+\noindent
+The
+\emph on
+PtlNIStatus
+\emph default
+ function returns the value of a status register for the specified interface.
+ (See section\SpecialChar ~
+
+\begin_inset LatexCommand \ref{sec:stat-type}
+
+\end_inset
+
+ for more information on status register indexes and status register values.)
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_INV_NI Indicates that
+\family typewriter
+interface
+\family default
+ is not a valid network interface handle.
+
+\layout Description
+
+PTL_INV_SR_INDX Indicates that
+\family typewriter
+status_register
+\family default
+ is not a valid status register.
+
+\layout Description
+
+PTL_SEGV Indicates that
+\family typewriter
+status
+\family default
+ is not a legal address.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="3" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="4.7in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+interface
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+A handle for the interface to use.
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+status_register
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+An index for the status register to read.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+status
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, this location will hold the current value of the status
+ register.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Comment
+
+The only status register that must be defined is a drop count register (
+\family typewriter
+PTL_SR_DROP_COUNT
+\family default
+).
+ Implementations may define additional status registers.
+ Identifiers for the indexes associated with these registers should start
+ with the prefix
+\family typewriter
+PTL_SR_
+\family default
+.
+\layout Subsection
+
+PtlNIDist
+\layout LyX-Code
+
+int PtlNIDist( ptl_handle_ni_t interface,
+\newline
+ ptl_process_id_t process,
+\newline
+ unsigned long* distance );
+\layout Standard
+\noindent
+The
+\emph on
+PtlNIDist
+\emph default
+ function returns the distance to another process using the specified interface.
+ Distances are only defined relative to an interface.
+ Distance comparisons between different interfaces on the same process may
+ be meaningless.
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_INV_NI Indicates that
+\family typewriter
+interface
+\family default
+ is not a valid network interface handle.
+
+\layout Description
+
+PTL_INV_PROC Indicates that
+\family typewriter
+process
+\family default
+ is not a valid process identifier.
+
+\layout Description
+
+PTL_SEGV Indicates that
+\family typewriter
+distance
+\family default
+ is not a legal address.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="3" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="4.7in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+interface
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+A handle for the interface to use.
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+process
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+An identifier for the process whose distance is being requested.
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+distance
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, this location will hold the distance to the remote
+ process.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Comment
+
+This function should return a static measure of distance.
+ Examples include minimum latency, the inverse of available bandwidth, or
+ the number of switches between the two endpoints.
+\layout Subsection
+
+PtlNIHandle
+\layout LyX-Code
+
+int PtlNIHandle( ptl_handle_any_t handle,
+\newline
+ ptl_handle_ni_t* interface );
+\layout Standard
+\noindent
+The
+\emph on
+PtlNIHandle
+\emph default
+ function returns a handle for the network interface with which the object
+ identified by
+\family typewriter
+handle
+\family default
+ is associated.
+ If the object identified by
+\family typewriter
+handle
+\family default
+ is a network interface, this function returns the same value it is passed.
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_INV_HANDLE Indicates that
+\family typewriter
+handle
+\family default
+ is not a valid handle.
+
+\layout Description
+
+PTL_SEGV Indicates that
+\family typewriter
+interface
+\family default
+ is not a legal address.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="2" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="4.7in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+handle
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+A handle for the object.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+interface
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, this location will hold a handle for the network interface
+ associated with
+\family typewriter
+handle
+\family default
+.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Comment
+
+Every handle should encode the network interface and the object id relative
+ to this handle.
+ Both are presumably encoded using integer values.
+\layout Section
+
+User Identification
+\begin_inset LatexCommand \label{sec:uid}
+
+\end_inset
+
+
+\layout Standard
+
+Every process runs on behalf of a user.
+
+\layout Subsection
+
+PtlGetUid
+\layout LyX-Code
+
+int PtlGetUid( ptl_handle_ni_t ni_handle,
+\newline
+ ptl_uid_t* uid );
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_INV_NI Indicates that
+\family typewriter
+ni_handle
+\family default
+ is not a valid network interface handle.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_SEGV Indicates that
+\family typewriter
+interface
+\family default
+ is not a legal address.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="2" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="5in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+handle
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+A network interface handle.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+id
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, this location will hold the user id for the calling
+ process.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Comment
+
+Note that user identifiers are dependent on the network interface(s).
+ In particular, if a node has multiple interfaces, a process may have multiple
+ user identifiers.
+\layout Section
+
+Process Identification
+\begin_inset LatexCommand \label{sec:pid}
+
+\end_inset
+
+
+\layout Standard
+
+Processes that use the Portals API, can be identified using a node id and
+ process id.
+ Every node accessible through a network interface has a unique node identifier
+ and every process running on a node has a unique process identifier.
+ As such, any process in the computing system can be identified by its node
+ id and process id.
+
+\layout Standard
+
+The Portals API defines a type,
+\family typewriter
+ptl_process_id_t
+\family default
+ for representing process ids and a function,
+\emph on
+PtlGetId
+\emph default
+, which can be used to obtain the id of the current process.
+\layout Comment
+
+The portals API does not include thread identifiers.
+ Messages are delivered to processes (address spaces) not threads (contexts
+ of execution).
+\layout Subsection
+
+The Process Id Type
+\begin_inset LatexCommand \label{sec:pid-type}
+
+\end_inset
+
+
+\layout LyX-Code
+
+typedef struct {
+\newline
+ ptl_nid_t nid; /* node id */
+\newline
+ ptl_pid_t pid; /* process id */
+\newline
+} ptl_process_id_t;
+\layout Standard
+\noindent
+The
+\family typewriter
+ptl_process_id_t
+\family default
+ type uses two identifiers to represent a process id: a node id and a process
+ id.
+
+\layout Subsection
+
+PtlGetId
+\begin_inset LatexCommand \label{sub:PtlGetId}
+
+\end_inset
+
+
+\layout LyX-Code
+
+int PtlGetId( ptl_handle_ni_t ni_handle,
+\newline
+ ptl_process_id_t* id );
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_INV_NI Indicates that
+\family typewriter
+ni_handle
+\family default
+ is not a valid network interface handle.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_SEGV Indicates that
+\family typewriter
+id
+\family default
+ is not a legal address.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="2" columns="3">
+<features>
+<column alignment="right" valignment="top" width="0pt">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="5in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+handle
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+A network interface handle.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\family typewriter
+id
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+output
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+On successful return, this location will hold the id for the calling process.
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\layout Comment
+
+Note that process identifiers are dependent on the network interface(s).
+ In particular, if a node has multiple interfaces, it may have multiple
+ node identifiers.
+\layout Section
+
+Match List Entries and Match Lists
+\begin_inset LatexCommand \label{sec:me}
+
+\end_inset
+
+
+\layout Standard
+
+A match list is a chain of match list entries.
+ Each match list entry includes a memory descriptor and a set of match criteria.
+ The match criteria can be used to reject incoming requests based on process
+ id or the match bits provided in the request.
+ A match list is created using the
+\emph on
+PtlMEAttach
+\emph default
+ or
+\shape italic
+PtlMEAttachAny
+\shape default
+ functions, which create a match list consisting of a single match list
+ entry, attaches the match list to the specified Portal index, and returns
+ a handle for the match list entry.
+ Match entries can be dynamically inserted and removed from a match list
+ using the
+\emph on
+PtlMEInsert
+\emph default
+ and
+\emph on
+PtlMEUnlink
+\emph default
+ functions.
+\layout Subsection
+
+PtlMEAttach
+\begin_inset LatexCommand \label{sec:meattach}
+
+\end_inset
+
+
+\layout LyX-Code
+
+typedef enum { PTL_RETAIN, PTL_UNLINK } ptl_unlink_t;
+\newline
+
+\layout LyX-Code
+
+typedef enum { PTL_INS_BEFORE, PTL_INS_AFTER } ptl_ins_pos_t;
+\newline
+
+\layout LyX-Code
+
+int PtlMEAttach( ptl_handle_ni_t interface,
+\newline
+ ptl_pt_index_t index,
+\newline
+ ptl_process_id_t matchid,
+\newline
+ ptl_match_bits_t match_bits,
+\newline
+ ptl_match_bits_t ignorebits,
+\newline
+ ptl_unlink_t unlink,
+\newline
+ ptl_ins_pos_t position,
+\newline
+ ptl_handle_me_t* handle );
+\layout Standard
+\noindent
+Values of the type
+\family typewriter
+ptl_ins_pos_t
+\family default
+ are used to control where a new item is inserted.
+ The value
+\family typewriter
+PTL_INS_BEFORE
+\family default
+ is used to insert the new item before the current item or before the head
+ of the list.
+ The value
+\family typewriter
+PTL_INS_AFTER
+\family default
+ is used to insert the new item after the current item or after the last
+ item in the list.
+
+\layout Standard
+
+The
+\emph on
+PtlMEAttach
+\emph default
+ function creates a match list consisting of a single entry and attaches
+ this list to the Portal table for
+\family typewriter
+interface
+\family default
+.
+\layout Subsubsection
+
+Return Codes
+\layout Description
+
+PTL_OK Indicates success.
+
+\layout Description
+
+PTL_INV_NI Indicates that
+\family typewriter
+interface
+\family default
+ is not a valid network interface handle.
+
+\layout Description
+
+PTL_NOINIT Indicates that the Portals API has not been successfully initialized.
+
+\layout Description
+
+PTL_INV_PTINDEX Indicates that
+\family typewriter
+index
+\family default
+ is not a valid Portal table index.
+
+\layout Description
+
+PTL_INV_PROC Indicates that
+\family typewriter
+matchid
+\family default
+ is not a valid process identifier.
+
+\layout Description
+
+PTL_NOSPACE Indicates that there is insufficient memory to allocate the
+ match list entry.
+
+\layout Description
+
+PTL_ML_TOOLONG Indicates that the resulting match list is too long.
+ The maximum length for a match list is defined by the interface.
+
+\layout Subsubsection
+
+Arguments
+\layout Standard
+
+
+\begin_inset Tabular
+<lyxtabular version="3" rows="7" columns="3">
+<features>
+<column alignment="left" valignment="top" width="0.8in">
+<column alignment="center" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="4.75in">
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+
+\family typewriter
+interface
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+A handle for the interface to use.
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+
+\family typewriter
+index
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+The Portal table index where the match list should be attached.
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+
+\family typewriter
+matchid
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+Specifies the match criteria for the process id of the requestor.
+ The constants
+\family typewriter
+PTL_PID_ANY
+\family default
+ and
+\family typewriter
+PTL_NID_ANY
+\family default
+ can be used to wildcard either of the ids in the
+\family typewriter
+ptl_process_id_t
+\family default
+ structure.
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+
+\family typewriter
+match_bits, ignorebits
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+
+
+\series bold
+input
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\layout Standard
+\noindent
+Specify the match criteria to apply to the&nbs
git://git.whamcloud.com / fs / lustre-release.git / commitdiff