991 lines
53 KiB
Clojure
991 lines
53 KiB
Clojure
{:highlights [{:id #uuid "63063a77-bddf-4e30-ab2a-5d009def0320",
|
|
:page 1,
|
|
:position {:bounding {:x1 234.16586303710938,
|
|
:y1 371.6640625,
|
|
:x2 307.5716857910156,
|
|
:y2 388.1640625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 234.16586303710938,
|
|
:y1 371.6640625,
|
|
:x2 307.5716857910156,
|
|
:y2 388.1640625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "BLEU score"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063a7c-1f1f-4833-9a37-20f185180632",
|
|
:page 1,
|
|
:position {:bounding {:x1 330.5127868652344,
|
|
:y1 371.6640625,
|
|
:x2 412.3282165527344,
|
|
:y2 388.1640625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 330.5127868652344,
|
|
:y1 371.6640625,
|
|
:x2 412.3282165527344,
|
|
:y2 388.1640625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text " n-gram-base"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063aa0-134b-43ac-a304-0de166e31190",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 590.84375,
|
|
:x2 490.08026123046875,
|
|
:y2 2209.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 116.8338623046875,
|
|
:y1 590.84375,
|
|
:x2 490.08026123046875,
|
|
:y2 607.34375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 609.1015625,
|
|
:x2 119.13267517089844,
|
|
:y2 625.6015625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2190,
|
|
:x2 0,
|
|
:y2 2209.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "allows for precisely and efficiently measuring the similarity of code"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063aae-e6f7-492b-ac1b-7c685e8771eb",
|
|
:page 1,
|
|
:position {:bounding {:x1 141.7746124267578,
|
|
:y1 627.3671875,
|
|
:x2 251.38177490234375,
|
|
:y2 643.8671875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 141.7746124267578,
|
|
:y1 627.3671875,
|
|
:x2 251.38177490234375,
|
|
:y2 643.8671875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text " language-agnosti"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063ab3-3683-42b4-ab50-23dd74f3704e",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 627.3671875,
|
|
:x2 490.4630126953125,
|
|
:y2 2241.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 307.3134765625,
|
|
:y1 627.3671875,
|
|
:x2 490.4630126953125,
|
|
:y2 643.8671875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 645.6328125,
|
|
:x2 178.50779724121094,
|
|
:y2 662.1328125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2222,
|
|
:x2 0,
|
|
:y2 2241.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text " handle incomplete or partially incorrect code"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063ab5-108f-4eaf-acd0-1b7334e6016e",
|
|
:page 1,
|
|
:position {:bounding {:x1 213.89752197265625,
|
|
:y1 645.6328125,
|
|
:x2 262.40301513671875,
|
|
:y2 662.1328125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 213.89752197265625,
|
|
:y1 645.6328125,
|
|
:x2 262.40301513671875,
|
|
:y2 662.1328125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "efficient"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063cad-79f4-45da-8134-64ac39b638da",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 983.03125,
|
|
:x2 490.0699462890625,
|
|
:y2 2513.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 364.5435485839844,
|
|
:y1 983.03125,
|
|
:x2 490.0699462890625,
|
|
:y2 999.53125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 1001.296875,
|
|
:x2 120.88430786132812,
|
|
:y2 1017.796875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2494,
|
|
:x2 0,
|
|
:y2 2513.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "automated program repai"},
|
|
:properties {:color "blue"}}
|
|
{:id #uuid "63063cb3-dc58-49e2-b55c-1fa1234f333c",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 1001.296875,
|
|
:x2 490.07861328125,
|
|
:y2 2529.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 264.674072265625,
|
|
:y1 1001.296875,
|
|
:x2 490.07861328125,
|
|
:y2 1017.796875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 1019.5625,
|
|
:x2 342.9353332519531,
|
|
:y2 1036.0625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2510,
|
|
:x2 0,
|
|
:y2 2529.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "predicting code from natural language descriptions of the desired functionality"},
|
|
:properties {:color "blue"}}
|
|
{:id #uuid "63063cd3-3c09-45f8-bf15-444de176fc9c",
|
|
:page 1,
|
|
:position {:bounding {:x1 161.2831268310547,
|
|
:y1 1129.15625,
|
|
:x2 357.9368896484375,
|
|
:y2 1145.65625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 161.2831268310547,
|
|
:y1 1129.15625,
|
|
:x2 357.9368896484375,
|
|
:y2 1145.65625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "bilingual evaluation understudy"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063d0f-c404-455d-91eb-9f7481e1db50",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 330.5625,
|
|
:x2 930.3394775390625,
|
|
:y2 2753.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 620.3577270507812,
|
|
:y1 330.5625,
|
|
:x2 930.3394775390625,
|
|
:y2 347.0625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 348.828125,
|
|
:x2 827.2161865234375,
|
|
:y2 365.328125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2734,
|
|
:x2 0,
|
|
:y2 2753.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "e find at least 21 papers published since 2015 that use BLEU as a metric to evaluate code prediction"},
|
|
:properties {:color "blue"}}
|
|
{:id #uuid "63063d27-0931-46bb-89ba-6436d127cdbf",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 367.09375,
|
|
:x2 930.3442993164062,
|
|
:y2 2801.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 546.5234375,
|
|
:y1 367.09375,
|
|
:x2 930.3284301757812,
|
|
:y2 383.59375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 385.359375,
|
|
:x2 930.3442993164062,
|
|
:y2 401.859375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 403.625,
|
|
:x2 843.0055541992188,
|
|
:y2 420.125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2766,
|
|
:x2 0,
|
|
:y2 2785.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2782,
|
|
:x2 0,
|
|
:y2 2801.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "The basic idea of BLEU is to compare a prediction against one or more reference solutions by computing the overlap between n-grams, i.e., contiguous sequences of code tokens"},
|
|
:properties {:color "blue"}}
|
|
{:id #uuid "63063df8-a6dc-4bee-96b2-2512719da00b",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 805.453125,
|
|
:x2 932.8565063476562,
|
|
:y2 3201.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 881.6500244140625,
|
|
:y1 805.453125,
|
|
:x2 930.3347778320312,
|
|
:y2 821.953125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 823.71875,
|
|
:x2 930.3419189453125,
|
|
:y2 840.21875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 841.984375,
|
|
:x2 932.8565063476562,
|
|
:y2 858.484375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 860.2421875,
|
|
:x2 635.2232666015625,
|
|
:y2 876.7421875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3150,
|
|
:x2 0,
|
|
:y2 3169.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3166,
|
|
:x2 0,
|
|
:y2 3185.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3182,
|
|
:x2 0,
|
|
:y2 3201.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "trivially shared n-grams, i.e., n-grams that occur across code written in the same language without implying any deeper relationship or semantic similarity. "},
|
|
:properties {:color "blue"}}
|
|
{:id #uuid "63063e31-2f9f-4a2b-9308-d98c5c9b71d3",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 933.3046875,
|
|
:x2 932.86328125,
|
|
:y2 3281.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 546.5234375,
|
|
:y1 933.3046875,
|
|
:x2 932.86328125,
|
|
:y2 949.8046875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 951.5703125,
|
|
:x2 932.6339721679688,
|
|
:y2 968.0703125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3262,
|
|
:x2 0,
|
|
:y2 3281.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "This paper presents CrystalBLEU, a metric to precisely and efficiently evaluate code similarity despite trivially shared n-grams."},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63063e49-ec06-4fb1-8a5f-567100bf353e",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 969.8359375,
|
|
:x2 930.3491821289062,
|
|
:y2 3329.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 529.4765625,
|
|
:y1 969.8359375,
|
|
:x2 930.3491821289062,
|
|
:y2 986.3359375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 988.1015625,
|
|
:x2 930.3449096679688,
|
|
:y2 1004.6015625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 1006.3671875,
|
|
:x2 574.51220703125,
|
|
:y2 1022.8671875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3294,
|
|
:x2 0,
|
|
:y2 3313.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3310,
|
|
:x2 0,
|
|
:y2 3329.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "The approach is an extension of BLEU that removes trivially shared n-grams before computing the n-gram overlap between two pieces of code"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "630642a3-0ec8-4822-8ae9-1c7c7f9ab432",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 1006.3671875,
|
|
:x2 930.3349609375,
|
|
:y2 3361.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 581.8124389648438,
|
|
:y1 1006.3671875,
|
|
:x2 930.0903930664062,
|
|
:y2 1022.8671875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 1024.6328125,
|
|
:x2 930.3349609375,
|
|
:y2 1041.1328125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 529.921875,
|
|
:y1 1042.890625,
|
|
:x2 629.4285278320312,
|
|
:y2 1059.390625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3326,
|
|
:x2 0,
|
|
:y2 3345.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 3342,
|
|
:x2 0,
|
|
:y2 3361.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "To identify trivially shared n-grams, CrystalBLEU analyzes a corpus of code and identifies n-grams that occur frequently across many examples"},
|
|
:properties {:color "blue"}}
|
|
{:id #uuid "630642b3-af01-4920-8ea7-a8e4fe046264",
|
|
:page 1,
|
|
:position {:bounding {:x1 826.8828125,
|
|
:y1 1152.484375,
|
|
:x2 929.4000854492188,
|
|
:y2 1168.984375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 826.8828125,
|
|
:y1 1152.484375,
|
|
:x2 929.4000854492188,
|
|
:y2 1168.984375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 1},
|
|
:content {:text "distinguishability"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "630642cc-fe7c-48e7-8633-ef1149f8f512",
|
|
:page 2,
|
|
:position {:bounding {:x1 0,
|
|
:y1 763.859375,
|
|
:x2 490.34727478027344,
|
|
:y2 1569.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 173.40538024902344,
|
|
:y1 763.859375,
|
|
:x2 490.34727478027344,
|
|
:y2 780.359375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 782.1171875,
|
|
:x2 490.06298828125,
|
|
:y2 798.6171875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 800.3828125,
|
|
:x2 423.3016357421875,
|
|
:y2 816.8828125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 1534,
|
|
:x2 0,
|
|
:y2 1553.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 1550,
|
|
:x2 0,
|
|
:y2 1569.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "distinguishability measures how much more similar code examples known to be semantically equivalent are compared to code examples that are not equivalent to each other."},
|
|
:properties {:color "green"}}
|
|
{:id #uuid "630642eb-1e39-4764-83c2-f36c4d42c4a1",
|
|
:page 2,
|
|
:position {:bounding {:x1 89.65625,
|
|
:y1 836.9140625,
|
|
:x2 140.1719512939453,
|
|
:y2 853.4140625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 89.65625,
|
|
:y1 836.9140625,
|
|
:x2 140.1719512939453,
|
|
:y2 853.4140625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "Concode"},
|
|
:properties {:color "red"}}
|
|
{:id #uuid "630642ee-9b86-4ec4-866e-d0143aa8d660",
|
|
:page 2,
|
|
:position {:bounding {:x1 103.3125,
|
|
:y1 873.4453125,
|
|
:x2 192.46713256835938,
|
|
:y2 889.9453125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 103.3125,
|
|
:y1 873.4453125,
|
|
:x2 192.46713256835938,
|
|
:y2 889.9453125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "BigCloneBench"},
|
|
:properties {:color "red"}}
|
|
{:id #uuid "63064327-6edb-4a1d-9e22-12f553aa941f",
|
|
:page 2,
|
|
:position {:bounding {:x1 0,
|
|
:y1 1019.5625,
|
|
:x2 490.0792236328125,
|
|
:y2 1793.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 358.3905334472656,
|
|
:y1 1019.5625,
|
|
:x2 490.0792236328125,
|
|
:y2 1036.0625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 1037.828125,
|
|
:x2 490.0673828125,
|
|
:y2 1054.328125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 89.65625,
|
|
:y1 1056.09375,
|
|
:x2 226.22259521484375,
|
|
:y2 1072.59375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 1758,
|
|
:x2 0,
|
|
:y2 1777.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 1774,
|
|
:x2 0,
|
|
:y2 1793.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "it is more effective at distinguishing semantically similar code from code merely written in the same language."},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "63064c06-55b1-45a7-bda0-b7c861303d12",
|
|
:page 2,
|
|
:position {:bounding {:x1 0,
|
|
:y1 659.359375,
|
|
:x2 932.8679809570312,
|
|
:y2 2417.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 720.821044921875,
|
|
:y1 659.359375,
|
|
:x2 932.8679809570312,
|
|
:y2 675.859375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 570.6015625,
|
|
:y1 677.6171875,
|
|
:x2 930.33642578125,
|
|
:y2 694.1171875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 570.6015625,
|
|
:y1 695.8828125,
|
|
:x2 932.8642578125,
|
|
:y2 712.3828125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 570.6015625,
|
|
:y1 714.1484375,
|
|
:x2 930.3360595703125,
|
|
:y2 730.6484375,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 570.6015625,
|
|
:y1 732.4140625,
|
|
:x2 631.545166015625,
|
|
:y2 748.9140625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2350,
|
|
:x2 0,
|
|
:y2 2369.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2366,
|
|
:x2 0,
|
|
:y2 2385.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2382,
|
|
:x2 0,
|
|
:y2 2401.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2398,
|
|
:x2 0,
|
|
:y2 2417.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "most common n-grams in programming languages appear relatively more often than the most common n-grams in natural languages, and that they appear in many different programs regardless of their semantic similarity."},
|
|
:properties {:color "green"}}
|
|
{:id #uuid "63064c0a-1933-4f6f-ae9b-04d65cafd1dd",
|
|
:page 2,
|
|
:position {:bounding {:x1 706.9219360351562,
|
|
:y1 750.6796875,
|
|
:x2 809.1270141601562,
|
|
:y2 767.1796875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 706.9219360351562,
|
|
:y1 750.6796875,
|
|
:x2 809.1270141601562,
|
|
:y2 767.1796875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "distinguishability"},
|
|
:properties {:color "green"}}
|
|
{:id #uuid "63064c19-dc64-4ece-b112-ddfbe89fc638",
|
|
:page 2,
|
|
:position {:bounding {:x1 0,
|
|
:y1 823.7421875,
|
|
:x2 932.8700561523438,
|
|
:y2 2529.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 883.1170043945312,
|
|
:y1 823.7421875,
|
|
:x2 932.068115234375,
|
|
:y2 840.2421875,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 570.6015625,
|
|
:y1 842.0078125,
|
|
:x2 932.8700561523438,
|
|
:y2 858.5078125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 570.6015625,
|
|
:y1 860.265625,
|
|
:x2 705.7455444335938,
|
|
:y2 876.765625,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2494,
|
|
:x2 0,
|
|
:y2 2513.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}
|
|
{:x1 0,
|
|
:y1 2510,
|
|
:x2 0,
|
|
:y2 2529.5,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "CrystalBLEU, which ignores trivially shared n-grams when comparing two pieces of code"},
|
|
:properties {:color "green"}}
|
|
{:id #uuid "63064e2b-f532-437f-a1a6-6466f735d592",
|
|
:page 2,
|
|
:position {:bounding {:x1 722.5859375,
|
|
:y1 1092.625,
|
|
:x2 832.8720092773438,
|
|
:y2 1109.125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998},
|
|
:rects ({:x1 722.5859375,
|
|
:y1 1092.625,
|
|
:x2 832.8720092773438,
|
|
:y2 1109.125,
|
|
:width 1019.9999999999999,
|
|
:height 1319.9999999999998}),
|
|
:page 2},
|
|
:content {:text "modified precision"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309dae9-1d7b-43c2-8f7b-2a5312d1b9c1",
|
|
:page 3,
|
|
:position {:bounding {:x1 269.448974609375,
|
|
:y1 362.484130859375,
|
|
:x2 346.8665771484375,
|
|
:y2 375.984130859375,
|
|
:width 816,
|
|
:height 1056},
|
|
:rects ({:x1 269.448974609375,
|
|
:y1 362.484130859375,
|
|
:x2 346.8665771484375,
|
|
:y2 375.984130859375,
|
|
:width 816,
|
|
:height 1056}),
|
|
:page 3},
|
|
:content {:text "odified precision"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309dafd-1db8-44d6-9e31-aef476b63017",
|
|
:page 4,
|
|
:position {:bounding {:x1 423.93109130859375,
|
|
:y1 726.484130859375,
|
|
:x2 542.3113403320312,
|
|
:y2 739.984130859375,
|
|
:width 816,
|
|
:height 1056},
|
|
:rects ({:x1 423.93109130859375,
|
|
:y1 726.484130859375,
|
|
:x2 542.3113403320312,
|
|
:y2 739.984130859375,
|
|
:width 816,
|
|
:height 1056}),
|
|
:page 4},
|
|
:content {:text "trivially shared n-grams,"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309dcb8-0ad1-4050-bc82-47e116305357",
|
|
:page 5,
|
|
:position {:bounding {:x1 0,
|
|
:y1 526,
|
|
:x2 53.142520904541016,
|
|
:y2 684.6360473632812,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 0,
|
|
:y1 526,
|
|
:x2 0,
|
|
:y2 545.5,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 53.135982513427734,
|
|
:y1 658.5579223632812,
|
|
:x2 53.142520904541016,
|
|
:y2 668.5579223632812,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 40.904537200927734,
|
|
:y1 674.6360473632812,
|
|
:x2 53.142520904541016,
|
|
:y2 684.6360473632812,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 5},
|
|
:content {:text "325"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e00a-1ea8-4ea1-aa59-94490799d9ec",
|
|
:page 5,
|
|
:position {:bounding {:x1 0,
|
|
:y1 774.0618896484375,
|
|
:x2 897.5899658203125,
|
|
:y2 2193.5,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 465.7023620605469,
|
|
:y1 774.0618896484375,
|
|
:x2 820.1432495117188,
|
|
:y2 789.0618896484375,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 448.794921875,
|
|
:y1 789.6723022460938,
|
|
:x2 897.5899658203125,
|
|
:y2 805.7348022460938,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 466.3211364746094,
|
|
:y1 806.2106323242188,
|
|
:x2 500.3293762207031,
|
|
:y2 821.2106323242188,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 0,
|
|
:y1 2158,
|
|
:x2 0,
|
|
:y2 2177.5,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 0,
|
|
:y1 2174,
|
|
:x2 0,
|
|
:y2 2193.5,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 5},
|
|
:content {:text "We now present the CrystalBLEU metric, which in contrast to BLEU, increases distinguishability by accounting for trivially shared ngrams"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e00f-9a1a-46dd-954c-0a7eb0e906bf",
|
|
:page 5,
|
|
:position {:bounding {:x1 494.4700012207031,
|
|
:y1 822.2809448242188,
|
|
:x2 700.743896484375,
|
|
:y2 837.2809448242188,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 494.4700012207031,
|
|
:y1 822.2809448242188,
|
|
:x2 700.743896484375,
|
|
:y2 837.2809448242188,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 5},
|
|
:content {:text "we identify trivially shared n-grams. "},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e012-ff0e-45bb-acee-870461906a76",
|
|
:page 5,
|
|
:position {:bounding {:x1 509.0059814453125,
|
|
:y1 838.3512573242188,
|
|
:x2 817.3226928710938,
|
|
:y2 853.3512573242188,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 509.0059814453125,
|
|
:y1 838.3512573242188,
|
|
:x2 817.3226928710938,
|
|
:y2 853.3512573242188,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 5},
|
|
:content {:text "BLEU score that accounts for the trivially shared n-grams"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e040-49ed-4a07-ab50-74b7174ff025",
|
|
:page 6,
|
|
:position {:bounding {:x1 0,
|
|
:y1 740.89794921875,
|
|
:x2 455.1771545410156,
|
|
:y2 1537.5,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 110.9140625,
|
|
:y1 740.89794921875,
|
|
:x2 455.1771545410156,
|
|
:y2 760.06201171875,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 110.90692138671875,
|
|
:y1 760.5231323242188,
|
|
:x2 197.16546630859375,
|
|
:y2 775.5231323242188,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}
|
|
{:x1 0,
|
|
:y1 1518,
|
|
:x2 0,
|
|
:y2 1537.5,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 6},
|
|
:content {:text "How well does CrystalBLEU distinguish between similar and dissimilar code?"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e043-4d6e-41d4-abb2-6ff76202290b",
|
|
:page 6,
|
|
:position {:bounding {:x1 204.98782348632812,
|
|
:y1 776.6005859375,
|
|
:x2 431.8476867675781,
|
|
:y2 791.6005859375,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 204.98782348632812,
|
|
:y1 776.6005859375,
|
|
:x2 431.8476867675781,
|
|
:y2 791.6005859375,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 6},
|
|
:content {:text "avoid misleading results provided by BLEU?"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e049-b50a-4077-8946-7ef78d2d86f7",
|
|
:page 6,
|
|
:position {:bounding {:x1 480.9432067871094,
|
|
:y1 922.6087646484375,
|
|
:x2 792.3610229492188,
|
|
:y2 937.6087646484375,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 480.9432067871094,
|
|
:y1 922.6087646484375,
|
|
:x2 792.3610229492188,
|
|
:y2 937.6087646484375,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 6},
|
|
:content {:text "ShareCode: Semantically equivalent, human-written code."},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e069-4d1b-4350-b0fe-648bcd92100d",
|
|
:page 7,
|
|
:position {:bounding {:x1 78.89295959472656,
|
|
:y1 331.57000732421875,
|
|
:x2 141.53402709960938,
|
|
:y2 346.57000732421875,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 78.89295959472656,
|
|
:y1 331.57000732421875,
|
|
:x2 141.53402709960938,
|
|
:y2 346.57000732421875,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 7},
|
|
:content {:text "ShareCode"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e07b-2a59-4dd1-930a-779695afda54",
|
|
:page 7,
|
|
:position {:bounding {:x1 93.51419830322266,
|
|
:y1 420.293212890625,
|
|
:x2 171.7705535888672,
|
|
:y2 435.293212890625,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 93.51419830322266,
|
|
:y1 420.293212890625,
|
|
:x2 171.7705535888672,
|
|
:y2 435.293212890625,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 7},
|
|
:content {:text "BigCloneBench"},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e085-79eb-4784-8423-63ad6e96d128",
|
|
:page 7,
|
|
:position {:bounding {:x1 93.5104751586914,
|
|
:y1 509.01580810546875,
|
|
:x2 260.05716705322266,
|
|
:y2 524.0158081054688,
|
|
:width 897.6,
|
|
:height 1161.6000000000001},
|
|
:rects ({:x1 93.5104751586914,
|
|
:y1 509.01580810546875,
|
|
:x2 260.05716705322266,
|
|
:y2 524.0158081054688,
|
|
:width 897.6,
|
|
:height 1161.6000000000001}),
|
|
:page 7},
|
|
:content {:text "Concode: Code generation task."},
|
|
:properties {:color "yellow"}}
|
|
{:id #uuid "6309e970-d42e-4be1-9735-9c239019a46a",
|
|
:page 1,
|
|
:position {:bounding {:x1 0,
|
|
:y1 112.28001403808594,
|
|
:x2 696.3023986816406,
|
|
:y2 1873.5,
|
|
:width 854,
|
|
:height 1105.1764705882351},
|
|
:rects ({:x1 157.68081665039062,
|
|
:y1 112.28001403808594,
|
|
:x2 696.3023986816406,
|
|
:y2 138.78001403808594,
|
|
:width 854,
|
|
:height 1105.1764705882351}
|
|
{:x1 306.4972839355469,
|
|
:y1 140.07823181152344,
|
|
:x2 547.4995727539062,
|
|
:y2 166.57823181152344,
|
|
:width 854,
|
|
:height 1105.1764705882351}
|
|
{:x1 0,
|
|
:y1 1854,
|
|
:x2 0,
|
|
:y2 1873.5,
|
|
:width 854,
|
|
:height 1105.1764705882351}),
|
|
:page 1},
|
|
:content {:text "CrystalBLEU: Precisely and Efficiently Measuring the Similarity of Code"},
|
|
:properties {:color "yellow"}}],
|
|
:extra {:page 1}}
|