Skip to content

Commit

Permalink
Merge branch 'microsoft:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@HelloBroBro
HelloBroBro committed Mar 20, 2024
2 parents 5e2cdd6 + 7ed98ba commit fe12f75
Show file tree
Hide file tree
Showing 3 changed files with 152 additions and 179 deletions.
32 changes: 9 additions & 23 deletions docs/examples/multimodal.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,32 +28,18 @@ print(pb.SUPPORTED_DATASETS_VLM)
dataset = pb.DatasetLoader.load_dataset("mmmu")

# print the first 5 examples
dataset[:5]
for idx in range(5):
print(dataset[idx])
```

All supported datasets:
['vqav2', 'nocaps', 'science_qa', 'math_vista', 'ai2d', 'mmmu', 'chart_qa']





[{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=733x237>],
'answer': 'B',
'question': '<image 1> Baxter Company has a relevant range of production between 15,000 and 30,000 units. The following cost data represents average variable costs per unit for 25,000 units of production. If 30,000 units are produced, what are the per unit manufacturing overhead costs incurred?\nA: $6\nB: $7\nC: $8\nD: $9'},
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=342x310>],
'answer': 'C',
'question': 'Assume accounts have normal balances, solve for the one missing account balance: Dividends. Equipment was recently purchased, so there is neither depreciation expense nor accumulated depreciation. <image 1>\nA: $194,815\nB: $182,815\nC: $12,000\nD: $9,000'},
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=336x169>],
'answer': 'B',
'question': 'Maxwell Software, Inc., has the following mutually exclusive projects.Suppose the company uses the NPV rule to rank these two projects.<image 1> Which project should be chosen if the appropriate discount rate is 15 percent?\nA: Project A\nB: Project B'},
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1222x237>],
'answer': 'D',
'question': "Each situation below relates to an independent company's Owners' Equity. <image 1> Calculate the missing values of company 2.\nA: $1,620\nB: $12,000\nC: $51,180\nD: $0"},
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1219x217>],
'answer': 'B',
'question': 'The following data show the units in beginning work in process inventory, the number of units started, the number of units transferred, and the percent completion of the ending work in process for conversion. Given that materials are added at the beginning of the process, what are the equivalent units for conversion costs for each quarter using the weighted-average method? Assume that the quarters are independent.<image 1>\nA: 132,625\nB: 134,485\nC: 135,332\nD: 132,685'}]

Images already saved to local, loading file: /home/v-mingxia/promptbench/promptbench/data/mmmu/validation.json
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=733x237 at 0x7F13BA2CD160>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/0_image_1.png'], 'answer': 'B', 'question': '<image 1> Baxter Company has a relevant range of production between 15,000 and 30,000 units. The following cost data represents average variable costs per unit for 25,000 units of production. If 30,000 units are produced, what are the per unit manufacturing overhead costs incurred?\nA: $6\nB: $7\nC: $8\nD: $9'}
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=342x310 at 0x7F13BA2CD550>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/1_image_1.png'], 'answer': 'C', 'question': 'Assume accounts have normal balances, solve for the one missing account balance: Dividends. Equipment was recently purchased, so there is neither depreciation expense nor accumulated depreciation. <image 1>\nA: $194,815\nB: $182,815\nC: $12,000\nD: $9,000'}
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=336x169 at 0x7F13BA2CD130>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/2_image_1.png'], 'answer': 'B', 'question': 'Maxwell Software, Inc., has the following mutually exclusive projects.Suppose the company uses the NPV rule to rank these two projects.<image 1> Which project should be chosen if the appropriate discount rate is 15 percent?\nA: Project A\nB: Project B'}
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1222x237 at 0x7F13BA2CD460>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/3_image_1.png'], 'answer': 'D', 'question': "Each situation below relates to an independent company's Owners' Equity. <image 1> Calculate the missing values of company 2.\nA: $1,620\nB: $12,000\nC: $51,180\nD: $0"}
{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1219x217 at 0x7F13BA2CD400>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/4_image_1.png'], 'answer': 'B', 'question': 'The following data show the units in beginning work in process inventory, the number of units started, the number of units transferred, and the percent completion of the ending work in process for conversion. Given that materials are added at the beginning of the process, what are the equivalent units for conversion costs for each quarter using the weighted-average method? Assume that the quarters are independent.<image 1>\nA: 132,625\nB: 134,485\nC: 135,332\nD: 132,685'}


## Load models
Expand Down Expand Up @@ -104,7 +90,7 @@ for prompt in prompts:
for data in tqdm(dataset):
# process input
input_text = pb.InputProcess.basic_format(prompt, data)
input_images = data['images']
input_images = data['images'] # please use data['image_paths'] instead of data['images'] for models that only support image path/url, such as GPT-4v
label = data['answer']
raw_pred = model(input_images, input_text)
# process output
Expand Down
37 changes: 10 additions & 27 deletions examples/multimodal.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,32 +51,14 @@
"output_type": "stream",
"text": [
"All supported datasets: \n",
"['vqav2', 'nocaps', 'science_qa', 'math_vista', 'ai2d', 'mmmu', 'chart_qa']\n"
"['vqav2', 'nocaps', 'science_qa', 'math_vista', 'ai2d', 'mmmu', 'chart_qa']\n",
"Images already saved to local, loading file: /home/v-mingxia/promptbench/promptbench/data/mmmu/validation.json\n",
"{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=733x237 at 0x7F13BA2CD160>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/0_image_1.png'], 'answer': 'B', 'question': '<image 1> Baxter Company has a relevant range of production between 15,000 and 30,000 units. The following cost data represents average variable costs per unit for 25,000 units of production. If 30,000 units are produced, what are the per unit manufacturing overhead costs incurred?\\nA: $6\\nB: $7\\nC: $8\\nD: $9'}\n",
"{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=342x310 at 0x7F13BA2CD550>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/1_image_1.png'], 'answer': 'C', 'question': 'Assume accounts have normal balances, solve for the one missing account balance: Dividends. Equipment was recently purchased, so there is neither depreciation expense nor accumulated depreciation. <image 1>\\nA: $194,815\\nB: $182,815\\nC: $12,000\\nD: $9,000'}\n",
"{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=336x169 at 0x7F13BA2CD130>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/2_image_1.png'], 'answer': 'B', 'question': 'Maxwell Software, Inc., has the following mutually exclusive projects.Suppose the company uses the NPV rule to rank these two projects.<image 1> Which project should be chosen if the appropriate discount rate is 15 percent?\\nA: Project A\\nB: Project B'}\n",
"{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1222x237 at 0x7F13BA2CD460>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/3_image_1.png'], 'answer': 'D', 'question': \"Each situation below relates to an independent company's Owners' Equity. <image 1> Calculate the missing values of company 2.\\nA: $1,620\\nB: $12,000\\nC: $51,180\\nD: $0\"}\n",
"{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1219x217 at 0x7F13BA2CD400>], 'image_paths': ['/home/v-mingxia/promptbench/promptbench/data/mmmu/validation/4_image_1.png'], 'answer': 'B', 'question': 'The following data show the units in beginning work in process inventory, the number of units started, the number of units transferred, and the percent completion of the ending work in process for conversion. Given that materials are added at the beginning of the process, what are the equivalent units for conversion costs for each quarter using the weighted-average method? Assume that the quarters are independent.<image 1>\\nA: 132,625\\nB: 134,485\\nC: 135,332\\nD: 132,685'}\n"
]
},
{
"data": {
"text/plain": [
"[{'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=733x237>],\n",
" 'answer': 'B',\n",
" 'question': '<image 1> Baxter Company has a relevant range of production between 15,000 and 30,000 units. The following cost data represents average variable costs per unit for 25,000 units of production. If 30,000 units are produced, what are the per unit manufacturing overhead costs incurred?\\nA: $6\\nB: $7\\nC: $8\\nD: $9'},\n",
" {'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=342x310>],\n",
" 'answer': 'C',\n",
" 'question': 'Assume accounts have normal balances, solve for the one missing account balance: Dividends. Equipment was recently purchased, so there is neither depreciation expense nor accumulated depreciation. <image 1>\\nA: $194,815\\nB: $182,815\\nC: $12,000\\nD: $9,000'},\n",
" {'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=336x169>],\n",
" 'answer': 'B',\n",
" 'question': 'Maxwell Software, Inc., has the following mutually exclusive projects.Suppose the company uses the NPV rule to rank these two projects.<image 1> Which project should be chosen if the appropriate discount rate is 15 percent?\\nA: Project A\\nB: Project B'},\n",
" {'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1222x237>],\n",
" 'answer': 'D',\n",
" 'question': \"Each situation below relates to an independent company's Owners' Equity. <image 1> Calculate the missing values of company 2.\\nA: $1,620\\nB: $12,000\\nC: $51,180\\nD: $0\"},\n",
" {'images': [<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1219x217>],\n",
" 'answer': 'B',\n",
" 'question': 'The following data show the units in beginning work in process inventory, the number of units started, the number of units transferred, and the percent completion of the ending work in process for conversion. Given that materials are added at the beginning of the process, what are the equivalent units for conversion costs for each quarter using the weighted-average method? Assume that the quarters are independent.<image 1>\\nA: 132,625\\nB: 134,485\\nC: 135,332\\nD: 132,685'}]"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Expand All @@ -89,7 +71,8 @@
"dataset = pb.DatasetLoader.load_dataset(\"mmmu\")\n",
"\n",
"# print the first 5 examples\n",
"dataset[:5]"
"for idx in range(5):\n",
" print(dataset[idx])"
]
},
{
Expand Down Expand Up @@ -219,7 +202,7 @@
" for data in tqdm(dataset):\n",
" # process input\n",
" input_text = pb.InputProcess.basic_format(prompt, data)\n",
" input_images = data['images']\n",
" input_images = data['images'] # please use data['image_paths'] instead of data['images'] for models that only support image path/url, such as GPT-4v\n",
" label = data['answer']\n",
" raw_pred = model(input_images, input_text)\n",
" # process output\n",
Expand Down
Loading

0 comments on commit fe12f75

Please sign in to comment.