{:highlights [{:id #uuid "63f15f7e-a63d-497e-bbc5-8480801e79e6", :page 1, :position {:bounding {:x1 0, :y1 62, :x2 440.0364990234375, :y2 261.72918701171875, :width 897.6, :height 1161.6000000000001}, :rects ({:x1 0, :y1 62, :x2 0, :y2 81.66667175292969, :width 897.6, :height 1161.6000000000001} {:x1 309.3417663574219, :y1 232.4479217529297, :x2 440.0364990234375, :y2 247.11459350585938, :width 897.6, :height 1161.6000000000001} {:x1 71.8125, :y1 247.0625, :x2 243.2283172607422, :y2 261.72918701171875, :width 897.6, :height 1161.6000000000001}), :page 1}, :content {:text "validate group fairness in image recognition software"}, :properties {:color "green"}} {:id #uuid "63f16015-e1e1-4eb5-94c7-a43da9c53d96", :page 1, :position {:bounding {:x1 0, :y1 142, :x2 440.0373229980469, :y2 378.61981201171875, :width 897.6, :height 1161.6000000000001}, :rects ({:x1 0, :y1 142, :x2 0, :y2 161.6666717529297, :width 897.6, :height 1161.6000000000001} {:x1 0, :y1 158, :x2 0, :y2 177.6666717529297, :width 897.6, :height 1161.6000000000001} {:x1 0, :y1 174, :x2 0, :y2 193.6666717529297, :width 897.6, :height 1161.6000000000001} {:x1 0, :y1 190, :x2 0, :y2 209.6666717529297, :width 897.6, :height 1161.6000000000001} {:x1 328.1458435058594, :y1 305.5052185058594, :x2 440.0293884277344, :y2 320.171875, :width 897.6, :height 1161.6000000000001} {:x1 71.8125, :y1 320.11981201171875, :x2 440.03662109375, :y2 334.7864685058594, :width 897.6, :height 1161.6000000000001} {:x1 71.8125, :y1 334.72918701171875, :x2 440.03472900390625, :y2 349.3958435058594, :width 897.6, :height 1161.6000000000001} {:x1 71.8125, :y1 349.34375, :x2 440.0373229980469, :y2 364.01043701171875, :width 897.6, :height 1161.6000000000001} {:x1 71.8125, :y1 363.953125, :x2 278.2251892089844, :y2 378.61981201171875, :width 897.6, :height 1161.6000000000001}), :page 1}, :content {:text "DISTROFAIR automatically learns the distribution (e.g., number/orientation) of objects in a set of images and systematically mutates objects in the images to become OOD using three semantic-preserving image mutations – object deletion,"}, :properties {:color "green"}} {:id #uuid "63f1cbbc-4a1a-491e-8465-8b576e2c949d", :page 1, :position {:bounding {:x1 0, :y1 286, :x2 480.06463623046875, :y2 508.77606201171875, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 0, :y1 286, :x2 0, :y2 305.66668701171875, :width 979.1999999999999, :height 1267.1999999999998} {:x1 363.6393737792969, :y1 476.8333435058594, :x2 480.06463623046875, :y2 492.8333435058594, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 492.77606201171875, :x2 140.04208374023438, :y2 508.77606201171875, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "class-level fairness violations"}, :properties {:color "yellow"}} {:id #uuid "63f1cbf5-4d2f-474f-90fc-a16954ce93b3", :page 1, :position {:bounding {:x1 196.66876220703125, :y1 636.3125, :x2 309.39630126953125, :y2 653.9791870117188, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 196.66876220703125, :y1 636.3125, :x2 309.39630126953125, :y2 653.9791870117188, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "image recognition"}, :properties {:color "green"}} {:id #uuid "63f1cc0d-033b-4d93-9a40-528da681b3c8", :page 1, :position {:bounding {:x1 0, :y1 510, :x2 480.03514099121094, :y2 787.8802185058594, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 0, :y1 510, :x2 0, :y2 529.6666717529297, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 526, :x2 0, :y2 545.6666717529297, :width 979.1999999999999, :height 1267.1999999999998} {:x1 346.9462585449219, :y1 731.953125, :x2 479.7742614746094, :y2 749.6198120117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 751.0833435058594, :x2 480.03514099121094, :y2 768.75, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 770.2135620117188, :x2 341.7386932373047, :y2 787.8802185058594, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text " systematic testing of image classification systems, to detect potential bias against certain classes, is of critical importance."}, :properties {:color "purple"}} {:id #uuid "63f1cc27-3991-4f6b-b669-631f7588433c", :page 1, :position {:bounding {:x1 0, :y1 590, :x2 480.0338134765625, :y2 882.828125, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 0, :y1 590, :x2 0, :y2 609.6666870117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 606, :x2 0, :y2 625.6666870117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 302.591552734375, :y1 826.90625, :x2 480.03179931640625, :y2 844.5729370117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 846.0364685058594, :x2 480.0338134765625, :y2 863.703125, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 865.1614990234375, :x2 189.22714233398438, :y2 882.828125, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "DISTROFAIR) identifies the classes that are subject to unfair treatment (i.e., unusually high error rates)"}, :properties {:color "green"}} {:id #uuid "63f1cc88-1792-40c0-b9d7-c7e83b282382", :page 1, :position {:bounding {:x1 0, :y1 686, :x2 480.0323944091797, :y2 997.5989685058594, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 0, :y1 686, :x2 0, :y2 705.6666717529297, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 702, :x2 0, :y2 721.6666717529297, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.73958587646484, :y1 941.6770935058594, :x2 480.0321044921875, :y2 959.34375, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 960.8020935058594, :x2 480.0323944091797, :y2 978.46875, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 979.9323120117188, :x2 384.74986267089844, :y2 997.5989685058594, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "DISTROFAIR learns the distribution of objects detected in a set of images and systematically generates a set of images, named OOD images, outside such a distributio"}, :properties {:color "green"}} {:id #uuid "63f1cfa9-832f-49b9-b6b2-13d64af880a5", :page 1, :position {:bounding {:x1 0, :y1 646.359375, :x2 900.8577728271484, :y2 1489.666748046875, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 596.8955688476562, :y1 646.359375, :x2 900.8573608398438, :y2 664.0260620117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 665.4895935058594, :x2 900.8577728271484, :y2 683.15625, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 684.6198120117188, :x2 811.3307342529297, :y2 702.2864685058594, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 1454, :x2 0, :y2 1473.666748046875, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 1470, :x2 0, :y2 1489.666748046875, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "OOD images are generated by leveraging this information and using semantic-preserving mutation operators(e.g., insertion, deletion and rotation of objects)."}, :properties {:color "yellow"}} {:id #uuid "63f1d03d-00a5-4dcc-81e9-361e5f340323", :page 1, :position {:bounding {:x1 0, :y1 960.1145935058594, :x2 900.8556518554688, :y2 1729.666748046875, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 688.5520324707031, :y1 960.1145935058594, :x2 900.8556518554688, :y2 977.78125, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 979.2395935058594, :x2 701.6606750488281, :y2 996.90625, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 1710, :x2 0, :y2 1729.666748046875, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "OOD images are about 80% as realistic as the original images. "}, :properties {:color "yellow"}} {:id #uuid "63f1d062-fcb8-41c9-9207-95dfb9626465", :page 1, :position {:bounding {:x1 0, :y1 1036.625, :x2 900.8550415039062, :y2 1793.666748046875, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 807.0899658203125, :y1 1036.625, :x2 900.8550415039062, :y2 1054.2916870117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 1055.7552185058594, :x2 581.8678588867188, :y2 1073.421875, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 1774, :x2 0, :y2 1793.666748046875, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "12K erroneous OOD image"}, :properties {:color "yellow"}} {:id #uuid "63f1d089-d192-4733-bce9-b0ad83884753", :page 1, :position {:bounding {:x1 499.16148376464844, :y1 1132.265625, :x2 679.603515625, :y2 1149.9323120117188, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 499.16148376464844, :y1 1132.265625, :x2 679.603515625, :y2 1149.9323120117188, :width 979.1999999999999, :height 1267.1999999999998}), :page 1}, :content {:text "discovery of fairness errors"}, :properties {:color "green"}} {:id #uuid "63f1d110-b8cd-475f-9e04-8b204ac2c7cd", :page 2, :position {:bounding {:x1 0, :y1 632.3999938964844, :x2 680.1453857421875, :y2 1168.0476379394531, :width 1387.2, :height 1795.2}, :rects ({:x1 0, :y1 632.3999938964844, :x2 0, :y2 654.6889038085938, :width 1387.2, :height 1795.2} {:x1 0, :y1 650.5333404541016, :x2 0, :y2 672.8222351074219, :width 1387.2, :height 1795.2} {:x1 619.287841796875, :y1 1088.5430603027344, :x2 680.1453857421875, :y2 1113.8542175292969, :width 1387.2, :height 1795.2} {:x1 110.9840316772461, :y1 1115.6368103027344, :x2 680.1436386108398, :y2 1140.9479675292969, :width 1387.2, :height 1795.2} {:x1 110.9840316772461, :y1 1142.7365417480469, :x2 620.9960250854492, :y2 1168.0476379394531, :width 1387.2, :height 1795.2}), :page 2}, :content {:text "lasses exhibiting an error rate higher than mean error rate across all classes would then be tagged as being unfairly treated."}, :properties {:color "green"}} {:id #uuid "63f1d113-6008-4087-8c2c-a6b7027135d8", :page 2, :position {:bounding {:x1 0, :y1 632.3999938964844, :x2 680.1453857421875, :y2 1168.0476379394531, :width 1387.2, :height 1795.2}, :rects ({:x1 0, :y1 632.3999938964844, :x2 0, :y2 654.6889038085938, :width 1387.2, :height 1795.2} {:x1 0, :y1 650.5333404541016, :x2 0, :y2 672.8222351074219, :width 1387.2, :height 1795.2} {:x1 603.787353515625, :y1 1088.5430603027344, :x2 680.1453857421875, :y2 1113.8542175292969, :width 1387.2, :height 1795.2} {:x1 110.9840316772461, :y1 1115.6368103027344, :x2 680.1436386108398, :y2 1140.9479675292969, :width 1387.2, :height 1795.2} {:x1 110.9840316772461, :y1 1142.7365417480469, :x2 615.1887741088867, :y2 1168.0476379394531, :width 1387.2, :height 1795.2}), :page 2}, :content {:text "Classes exhibiting an error rate higher than mean error rate across all classes would then be tagged as being unfairly treated"}, :properties {:color "green"}} {:id #uuid "63f1d116-a003-43fe-883d-cd87098c8183", :page 2, :position {:bounding {:x1 0, :y1 632.3999938964844, :x2 680.1453857421875, :y2 1168.0476379394531, :width 1387.2, :height 1795.2}, :rects ({:x1 0, :y1 632.3999938964844, :x2 0, :y2 654.6889038085938, :width 1387.2, :height 1795.2} {:x1 0, :y1 650.5333404541016, :x2 0, :y2 672.8222351074219, :width 1387.2, :height 1795.2} {:x1 619.287841796875, :y1 1088.5430603027344, :x2 680.1453857421875, :y2 1113.8542175292969, :width 1387.2, :height 1795.2} {:x1 110.9840316772461, :y1 1115.6368103027344, :x2 680.1436386108398, :y2 1140.9479675292969, :width 1387.2, :height 1795.2} {:x1 110.9840316772461, :y1 1142.7365417480469, :x2 620.9960250854492, :y2 1168.0476379394531, :width 1387.2, :height 1795.2}), :page 2}, :content {:text "lasses exhibiting an error rate higher than mean error rate across all classes would then be tagged as being unfairly treated."}, :properties {:color "green"}} {:id #uuid "63f1d13d-100d-4d73-8454-204ba1ce3d0a", :page 2, :position {:bounding {:x1 0, :y1 718.0000152587891, :x2 480.03509521484375, :y2 1015.6094055175781, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 0, :y1 718.0000152587891, :x2 0, :y2 737.6666870117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 345.21649169921875, :y1 978.8125305175781, :x2 480.03509521484375, :y2 996.4792175292969, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 997.9427185058594, :x2 308.99359130859375, :y2 1015.6094055175781, :width 979.1999999999999, :height 1267.1999999999998}), :page 2}, :content {:text "DISTROFAIR induce class-level group fairness violation."}, :properties {:color "green"}} {:id #uuid "63f1d146-8587-4e74-b403-5f0cedc9a67d", :page 2, :position {:bounding {:x1 329.828125, :y1 1036.625, :x2 480.0355224609375, :y2 1054.2916870117188, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 329.828125, :y1 1036.625, :x2 480.0355224609375, :y2 1054.2916870117188, :width 979.1999999999999, :height 1267.1999999999998}), :page 2}, :content {:text "OOD image mutation"}, :properties {:color "green"}} {:id #uuid "63f1d17d-2b45-44b5-82f0-897d300f06a9", :page 2, :position {:bounding {:x1 0, :y1 798, :x2 480.0303955078125, :y2 1111.6771240234375, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 0, :y1 798, :x2 0, :y2 817.6666717529297, :width 979.1999999999999, :height 1267.1999999999998} {:x1 424.2395935058594, :y1 1074.8802185058594, :x2 480.0303955078125, :y2 1092.546875, :width 979.1999999999999, :height 1267.1999999999998} {:x1 78.33854675292969, :y1 1094.0104370117188, :x2 205.15623474121094, :y2 1111.6771240234375, :width 979.1999999999999, :height 1267.1999999999998}), :page 2}, :content {:text "insertion mutation operation"}, :properties {:color "green"}} {:id #uuid "63f1d1a1-90e3-4f56-acfb-41d12664ca3d", :page 2, :position {:bounding {:x1 0, :y1 386.203125, :x2 900.8561401367188, :y2 961.6666870117188, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 604.5448608398438, :y1 386.203125, :x2 900.8561401367188, :y2 403.86981201171875, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 405.3333435058594, :x2 900.8561248779297, :y2 423, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 424.4583435058594, :x2 556.5158538818359, :y2 442.125, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 926, :x2 0, :y2 945.6666870117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 942, :x2 0, :y2 961.6666870117188, :width 979.1999999999999, :height 1267.1999999999998}), :page 2}, :content {:text "developer is more than two times likely (up to131%) to find class-level fairness errors with OOD mutations than ID. "}, :properties {:color "green"}} {:id #uuid "63f1d1dc-4632-42b7-b909-19f054d73cde", :page 2, :position {:bounding {:x1 0, :y1 667.4427185058594, :x2 900.8665161132812, :y2 1185.6666870117188, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 833.9920654296875, :y1 667.4427185058594, :x2 900.8665161132812, :y2 685.109375, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 686.5677185058594, :x2 900.8560028076172, :y2 704.234375, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 705.6979370117188, :x2 623.4532470703125, :y2 723.3645935058594, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 1150, :x2 0, :y2 1169.6666870117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 1166, :x2 0, :y2 1185.6666870117188, :width 979.1999999999999, :height 1267.1999999999998}), :page 2}, :content {:text "ystematic approach to discover class-level fairness violations in image classification task"}, :properties {:color "green"}} {:id #uuid "63f1d1ec-bc72-4ae3-b565-e46e612e9dce", :page 2, :position {:bounding {:x1 0, :y1 724.8229370117188, :x2 900.8556518554688, :y2 1217.6666870117188, :width 979.1999999999999, :height 1267.1999999999998}, :rects ({:x1 847.3507080078125, :y1 724.8229370117188, :x2 900.8556518554688, :y2 742.4895935058594, :width 979.1999999999999, :height 1267.1999999999998} {:x1 499.16148376464844, :y1 743.953125, :x2 704.9178466796875, :y2 761.6198120117188, :width 979.1999999999999, :height 1267.1999999999998} {:x1 0, :y1 1198, :x2 0, :y2 1217.6666870117188, :width 979.1999999999999, :height 1267.1999999999998}), :page 2}, :content {:text "emantic preserving mutation operation"}, :properties {:color "yellow"}}], :extra {:page 1}}