https://victordibia.com/component---src-components-post-layout-js-…p-matting-index-mdx-0c27e4a6882fa8c652dd.js

1"use strict";(self.webpackChunkvictordibia_com=self.webpackChunkvictordibia_com||[]).push([[6455],{63731:function(e,t,n){n.r(t),n.d(t,{default:function(){return v}});var a=n(91031),r=n(2248),o=(n(13058),n(57682)),l=n(19956);function i(e){const t=Object.assign({p:"p",sup:"sup",a:"a",span:"span",em:"em",h2:"h2",section:"section",ol:"ol",li:"li"},(0,a.R)(),e.components),{Tldr:n,ExtLink:i}=t;return i||s("ExtLink",!0),n||s("Tldr",!0),r.createElement(r.Fragment,null,r.createElement(t.p,null,"This post is a summary of the paper by Lin et al 2020",r.createElement(t.sup,null,r.createElement(t.a,{href:"#user-content-fn-1",id:"user-content-fnref-1","data-footnote-ref":!0,"aria-describedby":"footnote-label"},"1"))," where they proposes a two stage deep neural network model for real time segmentation of subjects from background."),"\n",r.createElement(t.span,{dangerouslySetInnerHTML:{__html:'<figure class="gatsby-resp-image-figure" style="margin-bottom: 8px; border-radius: 0.125rem">\n    <span\n      class="gatsby-resp-image-wrapper"\n      style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 850px; "\n    >\n      <a\n    class="gatsby-resp-image-link"\n    href="/static/10b2c9dfc69cf42f4edba11f4b19341b/26a94/screen.jpg"\n    style="display: block"\n    target="_blank"\n    rel="noopener"\n  >\n    <span\n    class="gatsby-resp-image-background-image"\n    style="padding-bottom: 51.64319248826291%; position: relative; bottom: 0; left: 0; background-image: url(\'data:image/jpeg;base64,/9j/2wBDABALDA4MChAODQ4SERATGCgaGBYWGDEjJR0oOjM9PDkzODdASFxOQERXRTc4UG1RV19iZ2hnPk1xeXBkeFxlZ2P/2wBDARESEhgVGC8aGi9jQjhCY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2P/wgARCAAKABQDASIAAhEBAxEB/8QAFwAAAwEAAAAAAAAAAAAAAAAAAAMEAv/EABUBAQEAAAAAAAAAAAAAAAAAAAAB/9oADAMBAAIQAxAAAAGTAqGEgn//xAAbEAABBAMAAAAAAAAAAAAAAAACAAESEyExM//aAAgBAQABBQIwaFclVguZaX//xAAWEQEBAQAAAAAAAAAAAAAAAAAAASH/2gAIAQMBAT8BrX//xAAUEQEAAAAAAAAAAAAAAAAAAAAQ/9oACAECAQE/AT//xAAcEAABAwUAAAAAAAAAAAAAAAABABAxAhIiQUL/2gAIAQEABj8CJt0qRB6Qyli3/8QAGxABAAMAAwEAAAAAAAAAAAAAAQARIRAxQaH/2gAIAQEAAT8hBh4pv2BQLQ3erjopi4n09T7eP//aAAwDAQACAAMAAAAQ9O//xAAXEQADAQAAAAAAAAAAAAAAAAAAAVER/9oACAEDAQE/EEhlH//EABYRAQEBAAAAAAAAAAAAAAAAAAEAEf/aAAgBAgEBPxAdsb//xAAcEAEBAAICAwAAAAAAAAAAAAABEQAhYXEQMUH/2gAIAQEAAT8QqgL9ZfuA5NuI3SdmADc00ibde+MfChOryY1orT4//9k=\'); background-size: cover; display: block;"\n  ></span>\n  <img\n        class="gatsby-resp-image-image"\n        alt="The technique employed is based on background matting, where an additional frame of the background is captured and used in recovering the alpha matte and the foreground layer. Source: Lin et al 2020."\n        title=""\n        src="/static/10b2c9dfc69cf42f4edba11f4b19341b/faddd/screen.jpg"\n        srcset="/static/10b2c9dfc69cf42f4edba11f4b19341b/6a7c0/screen.jpg 213w,\n/static/10b2c9dfc69cf42f4edba11f4b19341b/b9721/screen.jpg 425w,\n/static/10b2c9dfc69cf42f4edba11f4b19341b/faddd/screen.jpg 850w,\n/static/10b2c9dfc69cf42f4edba11f4b19341b/d5412/screen.jpg 1275w,\n/static/10b2c9dfc69cf42f4edba11f4b19341b/2b013/screen.jpg 1700w,\n/static/10b2c9dfc69cf42f4edba11f4b19341b/26a94/screen.jpg 2523w"\n        sizes="(max-width: 850px) 100vw, 850px"\n        style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n        loading="lazy"\n        decoding="async"\n      />\n  </a>\n    </span>\n    <figcaption class="gatsby-resp-image-figcaption">The technique employed is based on background matting, where an additional frame of the background is captured and used in recovering the alpha matte and the foreground layer. Source: Lin et al 2020.</figcaption>\n  </figure>'}}),"\n",r.createElement(n,{label:"Review TL;DR"},r.createElement("div",{className:"font-semibold"},"Pros"),r.createElement("ul",null,r.createElement("li",null," ",r.createElement(o.q,{icon:"check",color:"green"})," Proposed model handles hair and subject boundary details much better than current approaches (think how Zoom might crop out hair portions or fail when your hand is close to your face or some other occlusion etc)"),r.createElement("li",null," ",r.createElement(o.q,{icon:"check",color:"green"})," They improve speed/latency the state of the art for processing large images. Previous approaches that attempt fine grained segmentation achieve 8 fps on 512",r.createElement(t.em,null,"512 images (pretty much unsuable). Their approach achieves 60FPS on HD images (1920"),"1080) and 30FPS on 4K images (3840×2160 "),r.createElement("li",null," ",r.createElement(o.q,{icon:"check",color:"green"})," They achieve these speed gains by using a two stage network. First network downsamples the image and outputs matte predictions + error prediction map at a low resolution. The second network (a refinement network) uses the low resolution result and original image to generate high-resolution output (fine grained detail) for only select regions of the image. "),r.createElement("li",null," ",r.createElement(o.q,{icon:"check",color:"green"})," They compare their approach with several existing approaches and create a zoom plugin that pipes model output to zoom. "),r.createElement("li",null," ",r.createElement(o.q,{icon:"check",color:"green"})," They provide sample code to reproduce their results and allow experimentation via notebooks. ")),r.createElement("div",{className:"font-semibold"},"Cons"),r.createElement("ul",null,r.createElement("li",null," ",r.createElement(o.q,{icon:"check",color:"orange"})," - System requires specifying  background image to work well. This is not a huge issue but introduces a new step (selecting background image) that might interfere with usability."),r.createElement("li",null," ",r.createElement(o.q,{icon:"check",color:"orange"})," - The results (30FPS on HD images and 60FPS on HD images) are run on a GPU -  Nvidia RTX 2080 TI GPU. This suggests it might still be unusable on CPUs (the majority of user environments) ")),r.createElement("div",null," ",r.createElement(o.q,{icon:"pdf"}),r.createElement(i,{link:"https://arxiv.org/pdf/2012.07810.pdf",newTab:!0}," Download the paper ")," ")),"\n",r.createElement(t.p,null,"Its pretty remarkable what has been accomplished here. Future work might benefit from:"),"\n",r.createElement("ul",null,r.createElement("li",null,r.createElement(o.q,{icon:"check",color:"green"}),r.createElement(t.p,null,"Efforts to definitely would benefit from further optimizing for latency while\nmaintaining fine grained segmentation (e.g. using residual U-blocks as proposed\nin the ",r.createElement(l.Link,{to:"/blog/u-2-net"},"U2Net Paper"),". This might lead to usable\nFPS values on commodity CPU machines.")),r.createElement("li",null,r.createElement(o.q,{icon:"check",color:"green"}),r.createElement(t.p,null,"Efforts to optimize for usability by elmininating the background image requirement.\nE.g. by reframing the ML problem, we can train the network to jointly predict\nthe background (image completion) in addition to the alpha matte values, and\nleverage this knowledge in predicting better matte values. Ideally, this formulation\nwill utilize background images (as labels) during training but not require them\nduring inference."))),"\n",r.createElement(t.p,null,"Overall, well written paper and well produced video explaining their work."),"\n",r.createElement(t.p,null,"I will be updating this post as I experiment with the model itself."),"\n",r.createElement(t.h2,{id:"references",style:{position:"relative"}}

1,r.createElement(t.a,{href:"#references","aria-label":"references permalink",className:"anchor before"},r.createElement(t.span,{dangerouslySetInnerHTML:{__html:'<svg aria-hidden="true" focusable="false" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg>'}})),"References"),"\n",r.createElement(t.section,{"data-footnotes":!0,className:"footnotes"},r.createElement(t.h2,{className:"sr-only",id:"footnote-label"},"Footnotes"),"\n",r.createElement(t.ol,null,"\n",r.createElement(t.li,{id:"user-content-fn-1"},"\n",r.createElement(t.p,null,"Lin, S., Ryabtsev, A., Sengupta, S., Curless, B., Seitz, S., & Kemelmacher-Shlizerman, I. (2020). Real-Time High-Resolution Background Matting. arXiv preprint arXiv:2012.07810. CVPR 2021. ",r.createElement(t.a,{href:"#user-content-fnref-1","data-footnote-backref":!0,className:"data-footnote-backref","aria-label":"Back to content"},"↩")),"\n"),"\n"),"\n"))}var c=function(e){void 0===e&&(e={});const{wrapper:t}=Object.assign({},(0,a.R)(),e.components);return t?r.createElement(t,e,r.createElement(i,e)):i(e)};function s(e,t){throw new Error("Expected "+(t?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}var A=n(87677),m=n.n(A),d=n(57080),g=n(60751),p=n(84849),u=n(28679),h=n(16215),f=n(6085),b=n(43978),E=n(80149),w=n(8392);const k={a:e=>{let{href:t,children:n,...a}=e;return function(e){return void 0===e&&(e=""),/amzn\.to|[?&]tag=/.test(e)}(t)?r.createElement("a",Object.assign({href:t,target:"_blank",rel:"sponsored nofollow noopener noreferrer"},a),n):r.createElement("a",Object.assign({href:t},a),n)},ExtLink:p.cN,Link:l.Link,LinkIcon:u.q,Callout:p.bm,CalloutOrange:p.Ry,CalloutInfo:p.f,CalloutQuote:p.Pj,CalloutLink:p.Ly,RelatedArticle:p.in,Highlight:p.f4,Separator:p.wv,Tldr:p.d9},y=e=>{let{data:{mdx:t,ogImage:n,allMdx:o},pageContext:{next:i,previous:c,related:s,id:A},children:p}=e;return r.createElement(d.A,{activePage:"blog"},r.createElement(g.A,{blog:!0,title:t.frontmatter.title,description:t.excerpt,ogImage:n&&n.childImageSharp.fixed.src,datePublished:t.frontmatter.isoDate,keywords:t.frontmatter.tags}),r.createElement("div",{className:"post-page"},r.createElement("div",{className:"post-body"},r.createElement("article",{className:"prose sm:prose md:prose lg:prose min-w-0 max-w-none tracking-normal"},r.createElement("header",{className:"post-head"},r.createElement(h.YK,{to:"/blog",className:"post-back text-sm"},"All posts"),r.createElement(h.W8,{items:[t.frontmatter.date,t.fields.timeToRead.text],className:"post-meta"}),r.createElement("h1",{id:"introduction",className:"post-title"},t.frontmatter.title),t.frontmatter.description&&r.createElement("p",{className:"post-desc"},t.frontmatter.description),t.frontmatter.tags&&r.createElement("div",{className:"ui-pills post-tags"},t.frontmatter.tags.map((e=>r.createElement(l.Link,{key:e,to:`/blogtags/${m()(e)}/`,className:"no-underline"},r.createElement(h.ab,null,e)))))),r.createElement(a.x,{components:k},p),r.createElement(b.c,{previous:c,next:i})),t.tableOfContents&&!0===t.frontmatter.toc&&r.createElement("aside",{className:"post-aside"},r.createElement(E.A,{items:t.tableOfContents.items}))),r.createElement(w.A,{posts:s,id:A,all:o.edges}),r.createElement("div",{className:"ui-section"},r.createElement(f.G,null))))};function v(e){return r.createElement(y,e,r.createElement(c,e))}},57682:function(e,t,n){n.d(t,{q:function(){return a.q}});var a=n(28679)}}]);