-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathindex.html
More file actions
856 lines (726 loc) · 28.5 KB
/
Copy pathindex.html
File metadata and controls
856 lines (726 loc) · 28.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<script src="bootstrap.js"></script>
<script type="text/javascript" charset="utf-8" src="https://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script>
<!---
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
--->
<script src="load-mathjax.js" async></script>
<style type="text/css">
body {
font-family: "Titillium Web", "HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif;
font-weight: 300;
font-size: 17px;
margin-left: auto;
margin-right: auto;
}
@media screen and (min-width: 980px){
body {
width: 980px;
}
}
h1 {
font-weight:300;
line-height: 1.15em;
}
h2 {
font-size: 1.75em;
}
a:link,a:visited {
color: #5364cc;
text-decoration: none;
}
a:hover {
color: #208799;
}
h1 {
text-align: center;
}
h2,h3 {
text-align: left;
}
h1 {
font-size: 40px;
font-weight: 500;
}
h2 {
font-weight: 400;
margin: 16px 0px 4px 0px;
}
h3 {
font-weight: 600;
margin: 16px 0px 4px 0px;
}
.paper-title {
padding: 1px 0px 1px 0px;
}
section {
margin: 32px 0px 32px 0px;
text-align: justify;
clear: both;
}
.col-5 {
width: 20%;
float: left;
}
.move-down {
margin-top:1.2cm;
}
.col-4 {
width: 25%;
float: left;
}
.col-3 {
width: 33%;
float: left;
}
.col-2 {
width: 50%;
float: left;
}
.col-1 {
width: 100%;
float: left;
}
.author-row, .affil-row {
font-size: 26px;
}
.author-row-new {
text-align: center;
}
.author-row-new a {
display: inline-block;
font-size: 20px;
padding: 4px;
}
.author-row-new sup {
color: #313436;
font-size: 12px;
}
.affiliations-new {
font-size: 18px;
text-align: center;
width: 80%;
margin: 0 auto;
margin-bottom: 20px;
}
.row {
margin: 16px 0px 16px 0px;
}
.authors {
font-size: 26px;
}
.affiliatons {
font-size: 18px;
}
.affil-row {
margin-top: 18px;
}
.teaser {
max-width: 100%;
}
.text-center {
text-align: center;
}
.screenshot {
width: 256px;
border: 1px solid #ddd;
}
.screenshot-el {
margin-bottom: 16px;
}
hr {
height: 1px;
border: 0;
border-top: 1px solid #ddd;
margin: 0;
}
.material-icons {
vertical-align: -6px;
}
p {
line-height: 1.25em;
}
.caption {
font-size: 16px;
color: #666;
margin-top: 4px;
margin-bottom: 10px;
text-align: left;
}
video {
display: block;
margin: auto;
}
figure {
display: block;
margin: auto;
margin-top: 10px;
margin-bottom: 10px;
}
#bibtex pre {
font-size: 14px;
background-color: #eee;
padding: 16px;
}
.blue {
color: #2c82c9;
font-weight: bold;
}
.orange {
color: #d35400;
font-weight: bold;
}
.flex-row {
display: flex;
flex-flow: row wrap;
padding: 0;
margin: 0;
list-style: none;
}
.paper-btn-coming-soon {
position: relative;
top: 0;
left: 0;
}
.coming-soon {
position: absolute;
top: -15px;
right: -15px;
}
.center {
margin-left: 10.0%;
margin-right: 10.0%;
}
.paper-btn {
position: relative;
text-align: center;
display: inline-block;
margin: 8px;
padding: 8px 8px;
border-width: 0;
outline: none;
border-radius: 2px;
background-color: #5364cc;
color: white !important;
font-size: 20px;
width: 100px;
font-weight: 600;
}
.paper-btn-tapestry {
position: relative;
text-align: center;
display: inline-block;
margin: 8px;
padding: 8px 8px;
border-width: 0;
outline: none;
border-radius: 2px;
background-color: #5364cc;
color: white !important;
font-size: 20px;
width: 200px;
font-weight: 600;
}
.paper-btn-parent {
display: flex;
justify-content: center;
margin: 16px 0px;
}
.paper-btn:hover {
opacity: 0.85;
}
.container {
margin-left: auto;
margin-right: auto;
padding-left: 16px;
padding-right: 16px;
}
.venue {
font-size: 23px;
}
.topnav {
background-color: #EEEEEE;
overflow: hidden;
}
.topnav div {
max-width: 1070px;
margin: 0 auto;
}
.topnav a {
display: inline-block;
color: black;
text-align: center;
vertical-align: middle;
padding: 16px 16px;
text-decoration: none;
font-size: 18px;
}
.topnav img {
padding: 2px 0px;
width: 100%;
margin: 0.2em 0px 0.3em 0px;
vertical-align: middle;
}
pre {
font-size: 0.9em;
padding-left: 7px;
padding-right: 7px;
padding-top: 3px;
padding-bottom: 3px;
border-radius: 3px;
background-color: rgb(235, 235, 235);
overflow-x: auto;
}
.download-thumb {
display: flex;
}
@media only screen and (max-width: 620px) {
.download-thumb {
display: none;
}
}
.paper-stuff {
width: 50%;
font-size: 20px;
}
@media only screen and (max-width: 620px) {
.paper-stuff {
width: 100%;
}
}
* {
box-sizing: border-box;
}
.column {
text-align: center;
float: left;
width: 16.666%;
padding: 5px;
}
.column3 {
text-align: center;
float: left;
width: 33.333%;
padding: 5px;
}
.column4 {
text-align: center;
float: left;
width: 50%;
padding: 5px;
}
.column5 {
text-align: center;
float: left;
width: 20%;
padding: 5px;
}
.column10 {
text-align: center;
float: left;
width: 10%;
padding: 5px;
}
.border-right {
border-right: 1px solid black;
}
.border-bottom{
border-bottom: 1px solid black;
}
.row-center {
margin: 16px 0px 16px 0px;
text-align: center;
}
/* Clearfix (clear floats) */
.row::after {
content: "";
clear: both;
display: table;
}
.img-fluid {
max-width: 100%;
height: auto;
}
.figure-img {
margin-bottom: 0.5rem;
line-height: 1;
}
.rounded-circle {
border-radius: 50% !important;
}
/* Responsive layout - makes the three columns stack on top of each other instead of next to each other */
@media screen and (max-width: 500px) {
.column {
width: 100%;
}
}
@media screen and (max-width: 500px) {
.column3 {
width: 100%;
}
}
</style>
<link rel="stylesheet" href="bootstrap-grid.css">
<script type="text/javascript" src="../js/hidebib.js"></script>
<link href='https://fonts.googleapis.com/css?family=Titillium+Web:400,600,400italic,600italic,300,300italic' rel='stylesheet' type='text/css'>
<head>
<title> Reduce, Reuse, Recycle: Compositional Generation with Energy-Based Diffusion Models and MCMC</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta property="og:description" content="Reduce, Reuse, Recycle: Compositional Generation with Energy-Based Diffusion Models and MCMC"/>
<link href="https://fonts.googleapis.com/css2?family=Material+Icons" rel="stylesheet">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:creator" content="@du_yilun">
<meta name="twitter:title" content="Reduce, Reuse, Recycle: Compositional Generation with Energy-Based Diffusion Models and MCMC">
<meta name="twitter:description" content="">
<meta name="twitter:image" content="">
</head>
<body>
<div class="container">
<div class="paper-title">
<h1>
Reduce, Reuse, Recycle: Compositional Generation with Energy-Based Diffusion Models and MCMC
</div>
<div id="authors">
<center>
<div class="author-row-new">
<a href="https://yilundu.github.io/">Yilun Du<sup>1</sup></a>,
<a href="https://conormdurkan.github.io/">Conor Durkan<sup>2</sup></a>,
<a href="https://rstrudel.github.io/">Robin Strudel<sup>4</sup></a>,
<a href="https://scholar.google.com/citations?user=rRJ9wTJMUB8C&hl=en">Joshua B. Tenenbaum<sup>3</sup></a>,
<a href="https://benanne.github.io/about/">Sander Dieleman<sup>2</sup></a>,
<a href="https://scholar.google.com/citations?user=GgQ9GEkAAAAJ&hl=en">Rob Fergus<sup>2</sup></a>,
<a href="http://www.sohldickstein.com/">Jascha Sohl-Dickstein<sup>3</sup></a>,
<a href="https://www.stats.ox.ac.uk/~doucet/">Arnaud Doucet<sup>2</sup></a>,
<a href="http://www.cs.toronto.edu/~wgrathwohl/">Will Grathwohl<sup>2</sup></a>
</div>
</center>
<center>
<div class="affiliations">
<span><sup>1</sup> MIT</span>
<span><sup>2</sup> Google Deepmind</span>
<span><sup>3</sup> Google Brain</span>
<span><sup>4</sup> INRIA</span><br/>
</div>
<div class="affil-row">
<div class="venue text-center"><b>ICML 2023 </b></div>
</div>
</center>
<div style="clear: both">
<div class="paper-btn-parent">
<a class="paper-btn" href="https://arxiv.org/abs/2302.11552">
<span class="material-icons"> description </span>
Paper
</a>
<a class="paper-btn" href="https://colab.research.google.com/drive/1jvlzWMc6oo-TH1fYMl6hsOYfrcQj2rEs?usp=sharing">
<span class="material-icons"> code </span>
Colab
</a>
<a class="paper-btn-tapestry" href="https://colab.research.google.com/github/yilundu/reduce_reuse_recycle/blob/main/notebooks/image_tapestry.ipynb">
<span class="material-icons"> code </span>
Tapestry Colab
</a>
<a class="paper-btn" href="https://github.com/yilundu/reduce_reuse_recycle">
<span class="material-icons"> code </span>
Code
</a>
</div>
</div>
</div>
<section id="teaser-image">
<center>
<figure>
<video class="centered" width="80%" autoplay loop muted playsinline class="video-background " >
<source src="materials/teaser.m4v" type="video/mp4">
Your browser does not support the video tag.
</video>
</figure>
</center>
</section>
<section id="abstract"/>
<hr>
<h2>Abstract</h2>
<div class="flex-row">
<p>
Since their introduction, diffusion models have quickly become the prevailing approach to generative modeling in many domains. They can be interpreted as learning the gradients of a time-varying sequence of log-probability density functions. This interpretation has motivated classifier-based and classifier-free guidance as methods for post-hoc control of diffusion models. In this work, we build upon these ideas using the score-based interpretation of diffusion models, and explore alternative ways to condition, modify, and reuse diffusion models for tasks involving compositional generation and guidance. In particular, we investigate why certain types of composition fail using current techniques and present a number of solutions. We conclude that the sampler (not the model) is responsible for this failure and propose new samplers, inspired by MCMC, which enable successful compositional generation. Further, we propose an energy-based parameterization of diffusion models which enables the use of new compositional operators and more sophisticated, Metropolis-corrected samplers. Intriguingly we find these samplers lead to notable improvements in compositional generation across a wide variety of problems such as classifier-guided ImageNet modeling and compositional text-to-image generation.
</p>
</div>
</section>
<section id="method"/>
<hr>
<h2>Method</h2>
<div class="flex-row">
<p>
Existing diffusion models are often trained on massive datasets with a vast amounts of computational resources. In this paper, we explore and present tools on how we may utilize probabilistic composition of as an algebra to repurpose diffusion models, <b>without any finetuning</b>, for variety of downstream tasks.
<br>
<br>
Consider two probability distributions $q^1(x)$ and $q^2(x)$, each represented with a different diffusion model. Can we draw samples from the product distribution $q^{\textup{prod}}(x) \propto q^1(x)q^2(x)$ specified by each diffusion model? One potential solution is to note that the diffusion process encodes the noisy gradients of each distribution, letting us use the <a href="https://energy-based-model.github.io/Compositional-Visual-Generation-with-Composable-Diffusion-Models/">sum of the scores of each diffusion process</a> to compose these models. While this approach can be effective, it is not completely mathematically accurate. To correctly sample from a reverse diffusion corresponding to $q^{\textup{prod}}(x)$, at each noise timestep $t$, we must compute the score:
\[ \nabla_{x}\log \tilde{q}_{t}^{\textup{ prod}}(x_t) = \nabla_x\log \left(\int
dx_{0}
q^{1}(x_{0})q^2(x_{0})~ q(x_t|x_{0})\right).\]
Directly summing up the predicted scores of each separate diffusion model instead gives us the score:
\[ \nabla_{x}\log q_{t}^{\textup{prod}}(x_t)
=
\nabla_{x}\log \left(\int dx_0 q^1(x_0)q(x_t|x_0)\right)+\nabla_{x}\log \left(\int dx_0 q^2(x_0)q(x_t|x_0)\right).\]
When the $t > 0$, the above expressions are not equal, and thus sample from the incorrect reverse diffusion process for $q^{\textup{prod}}(x)$. To address this theoretical issue, we propose two methodological contributions to properly sample across a set of different compositions of diffusion models (see our paper for analysis of why other forms of composition fail):
</p>
<p>
<b>Sampling from Composed Diffusion Models using Annealed MCMC:</b> While the score estimate $\nabla_{x}\log q_{t}^{\textup{prod}}(x_t)$ does not correspond to the correct score estimate necessary to sample from the reverse diffusion process for $q_{t}^{\textup{prod}}(x_t)$, it does define a unnormalized probability distribution (EBM) at timestep t. The sequence of score estimates across different time points can then be seen as defining an annealed sequence of distributions starting from Gaussian
noise and evolving to our desired distribution $q_{t}^{\textup{prod}}(x_t)$.
Thus, we may still sample from $q_{t}^{\textup{prod}}(x_t)$ using <b>Annealed MCMC</b> procedure, where we initialize a sample from
Gaussian noise, and draw samples sequentially across different intermediate distributions by running multiple steps of MCMC
sampling initialized from samples from the previous distribution.
</p>
<p>
<b>Energy Based Diffusion Parameterization:</b> In practice, MCMC sampling does not correctly sample from underlying distribution without a Metropolis Adjustment step. With the typical score parameterization of diffusion models,
this is not possible, as there is no unnormalized density associated with the score field. Instead, we propose
to use an energy based parameterization of diffusion model, where at each timestep, our neural network predicts an <b>scalar energy value for each data point</b> and the utilizes the gradient of the energy with respect to the input as the score for the diffusion process. The predicted energy gives us an <b>unnormalized estimate of
the probability density</b>, enabling us to use Metropolis Adjustment in sampling. We further show that
the unnormalized estimate of the density enables us to do additional compositions with a diffusion model.
</p>
<p>
Below, we demonstrate results illustrating how we may use the above tools to re-purpose diffusion models in a variety of different settings.
</p>
</div>
</section>
<section id="results">
<hr>
<h2>2D Distribution Composition</h2>
<div class="flex-row">
<p>First, we illustrate how we may generate samples from compositions of two 2D distributions
$p_1(x)$ and $p_2(x)$, each represented using a seperately trained diffusion model. Here, we
illustrate two compositions, the product and mixture of the two distributions (see the paper for more examples). To enable the mixture of two distributions, we must use our energy-based diffusion parameterization,
as we need a unnormalized density estimate of the probability of the sample under each model.
</p>
</div>
<figure>
<a>
<img width="100%" src="materials/2d.png">
</a>
<p class="caption">
Our proposed techniques enable us to
take two separately trained diffusion models on distributions $p_1(x)$ and $p_2(x)$ and generate
samples from various compositions of these distributions.
</p> <br>
</figure>
</center>
<hr>
<h2>Object Generation</h2>
<div class="flex-row">
<p>We may repurpose a diffusion model which generates an object at a single position $y_i$ to generate multi-object scenes with $c$ objects by utilizing the following probability decomposition:
\[ \log p_\theta(x | y_1, \ldots, y_c) = \log p_\theta(x) + \sum_{i=1}^c \left( \log p_\theta(x | y_i) - \log p_\theta(x) \right). \]
Below, we illustrate the results when using this decomposition. Our probabilistic repurposing enables us to accurately generate 5
different cubes in a scene. The use of MCMC sampling to transition between annealed distributions
substantially improves the ability to generate multi-object scenes, with increasing number of steps
of MCMC improving the underlying compositional generation performance (right image).
</p>
</div>
<center>
<figure>
<a>
<img width="100%" src="materials/cube.png">
</a>
<p class="caption">
<b>a)</b> Using a HMC sampling procedure enables more accurate generation of multi-object scenes. <b>b)</b> The accuracy of sampling from composed distribution monotonically increases with the number of MCMC sets with Metropolis
Adjustment (enabled by a energy-based diffusion parameterization) significantly improving compositional performance.
</p> <br>
</figure>
</center>
<hr>
<h2>Class-Conditional Generation</h2>
<div class="flex-row">
<p>We may repurpose an unconditional diffusion for conditional image generation by utilizing the probability distribution corresponding to the gradient estimate:
\[ \nabla_x\log p_\theta(x | y, t) = \nabla_x\log p_\theta(x | t) + \nabla_x\log p_\theta(y | x, t). \]
To sample from this distribution, in classifer based guidance, this gradient estimate is directly
plugged into the reverse diffusion procedure. Below, we illustrate that conditional image sampling
can be substantially improved by using HMC to effectively transition between annealed distributions.
</p>
</div>
<center>
<figure>
<a>
<img width="100%" src="materials/classifier.png">
</a>
<p class="caption">
HMC sampling transitions enables more accurate classifier based generation of images
</p> <br>
</figure>
</center>
<hr>
<h2>Compositional Text-to-Image Generation</h2>
<div class="flex-row">
<p>We may also probabilistically repurpose a text-to-image model for compositional image generation. For example, given a complex scene description:
\[ \text{``A horse'' \texttt{AND} (``A sandy beach'' \texttt{OR} ``Grass plains'') \texttt{AND} (\texttt{NOT} ``Sunny'').} \]
We may utilize probability as an algebra to specify this complex scene description:
\[
p^{\text{comp}}_\theta(x|y_{\text{text}}) \propto \frac{p_\theta(x|\text{``A horse''})\left[\frac{1}{2}p_\theta(x|\text{``A sandy beach''}) + \frac{1}{2}p_\theta(x|\text{``Grass plains''})\right]}{p_\theta(x|\text{``Sunny''})^\alpha}.
\]
We may then sample from this composed distribution consisting of multiple instances of the
text-to-image model using the sampling procedures described above. We provide example illustrations of this procedure below:
</p>
</div>
<center>
<figure>
<a>
<img width="80%" src="materials/text2img.png">
</a>
<p class="caption">
By composing different energy-based text-to-image diffusion models we can generate complex combinations of
natural language descriptions.
</p> <br>
</figure>
</center>
<hr>
<h2> Image Tapestries</h2>
<div class="flex-row">
<p> While text-to-image models generate images given natural language prompts, it is difficult to control the spatial location of different content, and difficult to generate images at higher resolutions than used during training. We may further compose multiple overlapping text-to-image models, at a variety of scales, to construct an image tapestry with different specified content at different locations and scales.
</p>
</div>
<center>
<figure>
<video class="centered" width="80%" autoplay loop muted playsinline class="video-background " >
<source src="materials/teaser_spaceship.m4v" type="video/mp4">
Your browser does not support the video tag.
</video>
</figure>
</center>
</section>
<section id="related_projects">
<hr>
<h2>Related Projects</h2>
<br>
Check out a list of our related papers on compositional generation and energy based models. A full list can be found <a href="https://energy-based-model.github.io/Energy-based-Model-MIT/">here</a>!
<br>
<div class="row vspace-top">
<div class="col-sm-3">
<video width="100%" playsinline="" autoplay="" loop="" preload="" muted="">
<source src="materials/related/teaser_glide.mp4" type="video/mp4">
</video>
</div>
<div class="col-sm-9">
<div class="paper-title">
<a href="https://energy-based-model.github.io/Compositional-Visual-Generation-with-Composable-Diffusion-Models/">Compositional Visual Generation with Composable Diffusion Models</a>
</div>
<div>
We present a method to compose different diffusion models together, drawing on the close connection of
diffusion models with EBMs. We illustrate how compositional operators enable
the ability to composing multiple sets of objects together as well as generate images subject to
complex text prompts.
</div>
</div>
</div>
<div class="row vspace-top">
<div class="col-sm-3">
<div class="move-down">
<img src="materials/related/comp_cartoon.png" class="img-fluid" alt="comp_carton" style="width:100%">
</div>
</div>
<div class="col-sm-9">
<div class="paper-title">
<a href="https://energy-based-model.github.io/compositional-generation-inference/">Compositional Visual Generation with Energy Based Models</a>
</div>
<div>
We present a set of compositional operators that enable EBMs to exhibit <b>zero-shot compositional</b> visual generation, enabling us to compose visual concepts
(through operators of conjunction, disjunction, or negation) together in a zero-shot manner.
Our approach enables us to generate faces given a description
((Smiling AND Female) OR (NOT Smiling AND Male)) or to combine several different objects together.
</div>
</div>
</div>
<div class="row vspace-top">
<div class="col-sm-3">
<video width="100%" playsinline="" autoplay="" loop="" preload="" muted="">
<source src="materials/related/half.mp4" type="video/mp4">
</video>
</div>
<div class="col-sm-9">
<div class="paper-title">
<a href="https://openai.com/blog/energy-based-models/">Implicit Generation and Generalization with Energy Based Models</a>
</div>
We introduce a method to scale EBM training to generate high resolution images.
We propose to utilize Langevin dynamics, initialized from random noise, to iteratively
refine and denoise image samples. We further demonstrate unique properties of EBMs
such as compositionality, continual learning, and robustness.
<div>
</div>
</div>
</section>
<section id="paper">
<h2>Team</h2>
<div class="row">
<div class="column5">
<a href='https://yilundu.github.io/'>
<img src=./materials/people/yilun3.png class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Yilun Du </p>
<p class=institution>MIT</p>
</div>
<div class="column5">
<a href='https://yilundu.github.io/'>
<img src=./materials/people/conor.jpg class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Conor Durkan </p>
<p class=institution>Google Deepmind</p>
</div>
<div class="column5">
<a href='https://yilundu.github.io/'>
<img src=./materials/people/robin.png class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Robin Strudel </p>
<p class=institution>INRIA</p>
</div>
<div class="column5">
<a href='https://scholar.google.com/citations?user=rRJ9wTJMUB8C&hl=en'>
<img src=./materials/people/josh2.jpg class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Joshua Tenenbaum </p>
<p class=institution>MIT</p>
</div>
<div class="column5">
<a href="https://groups.csail.mit.edu/vision/torralbalab/">
<img src=./materials/people/sander.png class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Sander Dieleman </p>
<p class=institution>Google Deepmind</p>
</div>
</div>
<div class="row">
<div class="column10">
</div>
<div class="column5">
<a href='https://yilundu.github.io/'>
<img src=./materials/people/rob.png class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Rob Fergus </p>
<p class=institution>Google Deepmind</p>
</div>
<div class="column5">
<a href='https://yilundu.github.io/'>
<img src=./materials/people/jascha.png class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Jascha Sohl-Dickstein </p>
<p class=institution>Google Brain</p>
</div>
<div class="column5">
<a href='https://yilundu.github.io/'>
<img src=./materials/people/arnaud.jpg class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Arnaud Doucet </p>
<p class=institution>Google Deepmind</p>
</div>
<div class="column5">
<a href='https://scholar.google.com/citations?user=rRJ9wTJMUB8C&hl=en'>
<img src=./materials/people/will.jpg class="figure-img img-fluid rounded-circle" height=200px width=200px>
</a>
<p class=profname> Will Grathwohl </p>
<p class=institution>Google Deepmind</p>
</div>
</div>
</section>
<section>
This webpage template was recycled from <a href='https://nv-tlabs.github.io/LION/'>here</a>.
<center><p><a href='https://accessibility.mit.edu/'><b>Accessibility</b></a></p></center>
</section>
</div>
</body>
</html>