9. Research articles vs datasets in chemistry and materials science#

In this notebook, we show how to retrieve meta information about papers and datasets to make a comparison between the number of research articles published in chemistry and materials science (Web of Science Core Collection) and the number of datasets deposited in data repositories (DataCite and ZENODO) from 1996 to 2023.

9.1. Example of data collection using the Web of Science API#

import os
import pandas as pd
import requests
import urllib.parse
import json
import time
import matextract  # noqa: F401

9.1.1. Create a function to request data from Web of Science#

def wos_api_request(i, query, wos_data):
    request = requests.get(
        f"https://api.clarivate.com/apis/wos-starter/v1/documents?db=WOS&q=TS=({urllib.parse.quote(query)})&"
        f"&limit=50&page={i+1}",
        headers={"X-APIKey": os.environ["WOS_API_KEY"]},
    )
    try:
        for wos_record in request.json()["hits"]:
            wos_data.append(wos_record)
    except (
        requests.exceptions.ConnectionError,
        requests.exceptions.JSONDecodeError,
        KeyError,
    ):
        print(f"Resending WoS API request #{i+1}")
        time.sleep(100)
        wos_api_request(i, query, wos_data)

9.1.2. Create a function to get Article and Data Paper document types published between 1996 and 2023#

def process_data(wos_data):
    years = []

    for paper in wos_data:
        if "Article" in paper["types"] or "Data Paper" in paper["types"]:
            if 1996 <= int(paper["source"]["publishYear"]) <= 2023:
                pub_year = int(paper["source"]["publishYear"])
                if len(years) == 0:
                    years.append(
                        {
                            "publication_year": pub_year,
                            "wos_documents": 1,
                        }
                    )
                else:
                    for year in years:
                        if year["publication_year"] == pub_year:
                            year["wos_documents"] += 1
                            break
                    else:
                        years.append(
                            {
                                "publication_year": pub_year,
                                "wos_documents": 1,
                            }
                        )
    return years

9.1.3. Create a function to save the results in a .csv file#

def save_results(file_name, years):
    years.sort(reverse=True, key=lambda years: years["wos_documents"])

    df = pd.DataFrame(data=years)
    column_names = {
        "publication_year": "publication_year",
        "wos_documents": "wos_documents",
    }
    df.rename(columns=column_names, inplace=True)

    df.to_csv(f"./wos_results/{file_name}.csv", sep=";", index=False)

9.1.4. Example of data collection on “thermoelectric materials”#

As an example, we extract from the Web of Science Core Collection the papers published on thermoelectric materials.

wos_data = []
query = "thermoelectric AND materials"

We send an initial WoS API request to assess the number of requests required to extract all the papers in this topic. From the initial response, we extract the total number of records found.

initial_wos_response = requests.get(
    f"https://api.clarivate.com/apis/wos-starter/v1/documents?db=WOS&q=TS=({urllib.parse.quote(query)})&"
    f"&limit=50&page=1",
    headers={"X-APIKey": os.environ["WOS_API_KEY"]},
)
data = initial_wos_response.json()
print(f"Number of papers from the query ({query}): {data['metadata']['total']}")
Number of papers from the query (thermoelectric AND materials): 29704

With the total number of records, we calculate the number of requests required.

wos_requests_required = ((data["metadata"]["total"] - 1) // 50) + 1
print(f"Number of requests required: {wos_requests_required}")
Number of requests required: 595

Now we can extract the papers from WoS. This can take several minutes, depending on the total number of records to be collected.

for i in range(wos_requests_required + 1):
    wos_api_request(i, query, wos_data)
    print(
        f"{(((i + 1) * 100) / wos_requests_required):.1f}% of WoS API requests complete"
    )
0.2% of WoS API requests complete
0.3% of WoS API requests complete
0.5% of WoS API requests complete
0.7% of WoS API requests complete
0.8% of WoS API requests complete
1.0% of WoS API requests complete
1.2% of WoS API requests complete
1.3% of WoS API requests complete
1.5% of WoS API requests complete
1.7% of WoS API requests complete
1.8% of WoS API requests complete
2.0% of WoS API requests complete
2.2% of WoS API requests complete
2.4% of WoS API requests complete
2.5% of WoS API requests complete
2.7% of WoS API requests complete
2.9% of WoS API requests complete
3.0% of WoS API requests complete
3.2% of WoS API requests complete
3.4% of WoS API requests complete
3.5% of WoS API requests complete
3.7% of WoS API requests complete
3.9% of WoS API requests complete
4.0% of WoS API requests complete
4.2% of WoS API requests complete
4.4% of WoS API requests complete
4.5% of WoS API requests complete
4.7% of WoS API requests complete
4.9% of WoS API requests complete
5.0% of WoS API requests complete
5.2% of WoS API requests complete
5.4% of WoS API requests complete
5.5% of WoS API requests complete
5.7% of WoS API requests complete
5.9% of WoS API requests complete
6.1% of WoS API requests complete
6.2% of WoS API requests complete
6.4% of WoS API requests complete
6.6% of WoS API requests complete
6.7% of WoS API requests complete
6.9% of WoS API requests complete
7.1% of WoS API requests complete
7.2% of WoS API requests complete
7.4% of WoS API requests complete
7.6% of WoS API requests complete
7.7% of WoS API requests complete
7.9% of WoS API requests complete
8.1% of WoS API requests complete
8.2% of WoS API requests complete
8.4% of WoS API requests complete
8.6% of WoS API requests complete
8.7% of WoS API requests complete
8.9% of WoS API requests complete
9.1% of WoS API requests complete
9.2% of WoS API requests complete
9.4% of WoS API requests complete
9.6% of WoS API requests complete
9.7% of WoS API requests complete
9.9% of WoS API requests complete
10.1% of WoS API requests complete
10.3% of WoS API requests complete
10.4% of WoS API requests complete
10.6% of WoS API requests complete
10.8% of WoS API requests complete
10.9% of WoS API requests complete
11.1% of WoS API requests complete
11.3% of WoS API requests complete
11.4% of WoS API requests complete
11.6% of WoS API requests complete
11.8% of WoS API requests complete
11.9% of WoS API requests complete
12.1% of WoS API requests complete
12.3% of WoS API requests complete
12.4% of WoS API requests complete
12.6% of WoS API requests complete
12.8% of WoS API requests complete
12.9% of WoS API requests complete
13.1% of WoS API requests complete
13.3% of WoS API requests complete
13.4% of WoS API requests complete
13.6% of WoS API requests complete
13.8% of WoS API requests complete
13.9% of WoS API requests complete
14.1% of WoS API requests complete
14.3% of WoS API requests complete
14.5% of WoS API requests complete
14.6% of WoS API requests complete
14.8% of WoS API requests complete
15.0% of WoS API requests complete
15.1% of WoS API requests complete
15.3% of WoS API requests complete
15.5% of WoS API requests complete
15.6% of WoS API requests complete
15.8% of WoS API requests complete
16.0% of WoS API requests complete
16.1% of WoS API requests complete
16.3% of WoS API requests complete
16.5% of WoS API requests complete
16.6% of WoS API requests complete
16.8% of WoS API requests complete
17.0% of WoS API requests complete
17.1% of WoS API requests complete
17.3% of WoS API requests complete
17.5% of WoS API requests complete
17.6% of WoS API requests complete
17.8% of WoS API requests complete
18.0% of WoS API requests complete
18.2% of WoS API requests complete
18.3% of WoS API requests complete
18.5% of WoS API requests complete
18.7% of WoS API requests complete
18.8% of WoS API requests complete
19.0% of WoS API requests complete
19.2% of WoS API requests complete
19.3% of WoS API requests complete
19.5% of WoS API requests complete
19.7% of WoS API requests complete
19.8% of WoS API requests complete
20.0% of WoS API requests complete
20.2% of WoS API requests complete
20.3% of WoS API requests complete
20.5% of WoS API requests complete
20.7% of WoS API requests complete
20.8% of WoS API requests complete
21.0% of WoS API requests complete
21.2% of WoS API requests complete
21.3% of WoS API requests complete
21.5% of WoS API requests complete
21.7% of WoS API requests complete
21.8% of WoS API requests complete
22.0% of WoS API requests complete
22.2% of WoS API requests complete
22.4% of WoS API requests complete
22.5% of WoS API requests complete
22.7% of WoS API requests complete
22.9% of WoS API requests complete
23.0% of WoS API requests complete
23.2% of WoS API requests complete
23.4% of WoS API requests complete
23.5% of WoS API requests complete
23.7% of WoS API requests complete
23.9% of WoS API requests complete
24.0% of WoS API requests complete
24.2% of WoS API requests complete
24.4% of WoS API requests complete
24.5% of WoS API requests complete
24.7% of WoS API requests complete
24.9% of WoS API requests complete
25.0% of WoS API requests complete
25.2% of WoS API requests complete
25.4% of WoS API requests complete
25.5% of WoS API requests complete
25.7% of WoS API requests complete
25.9% of WoS API requests complete
26.1% of WoS API requests complete
26.2% of WoS API requests complete
26.4% of WoS API requests complete
26.6% of WoS API requests complete
26.7% of WoS API requests complete
26.9% of WoS API requests complete
27.1% of WoS API requests complete
27.2% of WoS API requests complete
27.4% of WoS API requests complete
27.6% of WoS API requests complete
27.7% of WoS API requests complete
27.9% of WoS API requests complete
28.1% of WoS API requests complete
28.2% of WoS API requests complete
28.4% of WoS API requests complete
28.6% of WoS API requests complete
28.7% of WoS API requests complete
28.9% of WoS API requests complete
29.1% of WoS API requests complete
29.2% of WoS API requests complete
29.4% of WoS API requests complete
29.6% of WoS API requests complete
29.7% of WoS API requests complete
29.9% of WoS API requests complete
30.1% of WoS API requests complete
30.3% of WoS API requests complete
30.4% of WoS API requests complete
30.6% of WoS API requests complete
30.8% of WoS API requests complete
30.9% of WoS API requests complete
31.1% of WoS API requests complete
31.3% of WoS API requests complete
31.4% of WoS API requests complete
31.6% of WoS API requests complete
31.8% of WoS API requests complete
31.9% of WoS API requests complete
32.1% of WoS API requests complete
32.3% of WoS API requests complete
32.4% of WoS API requests complete
32.6% of WoS API requests complete
32.8% of WoS API requests complete
32.9% of WoS API requests complete
33.1% of WoS API requests complete
33.3% of WoS API requests complete
33.4% of WoS API requests complete
33.6% of WoS API requests complete
33.8% of WoS API requests complete
33.9% of WoS API requests complete
34.1% of WoS API requests complete
34.3% of WoS API requests complete
34.5% of WoS API requests complete
34.6% of WoS API requests complete
34.8% of WoS API requests complete
35.0% of WoS API requests complete
35.1% of WoS API requests complete
35.3% of WoS API requests complete
35.5% of WoS API requests complete
35.6% of WoS API requests complete
35.8% of WoS API requests complete
36.0% of WoS API requests complete
36.1% of WoS API requests complete
36.3% of WoS API requests complete
36.5% of WoS API requests complete
36.6% of WoS API requests complete
36.8% of WoS API requests complete
37.0% of WoS API requests complete
37.1% of WoS API requests complete
37.3% of WoS API requests complete
37.5% of WoS API requests complete
37.6% of WoS API requests complete
37.8% of WoS API requests complete
38.0% of WoS API requests complete
38.2% of WoS API requests complete
38.3% of WoS API requests complete
38.5% of WoS API requests complete
38.7% of WoS API requests complete
38.8% of WoS API requests complete
39.0% of WoS API requests complete
39.2% of WoS API requests complete
39.3% of WoS API requests complete
39.5% of WoS API requests complete
39.7% of WoS API requests complete
39.8% of WoS API requests complete
40.0% of WoS API requests complete
40.2% of WoS API requests complete
40.3% of WoS API requests complete
40.5% of WoS API requests complete
40.7% of WoS API requests complete
40.8% of WoS API requests complete
41.0% of WoS API requests complete
41.2% of WoS API requests complete
41.3% of WoS API requests complete
41.5% of WoS API requests complete
41.7% of WoS API requests complete
41.8% of WoS API requests complete
42.0% of WoS API requests complete
42.2% of WoS API requests complete
42.4% of WoS API requests complete
42.5% of WoS API requests complete
42.7% of WoS API requests complete
42.9% of WoS API requests complete
43.0% of WoS API requests complete
43.2% of WoS API requests complete
43.4% of WoS API requests complete
43.5% of WoS API requests complete
43.7% of WoS API requests complete
43.9% of WoS API requests complete
44.0% of WoS API requests complete
44.2% of WoS API requests complete
44.4% of WoS API requests complete
44.5% of WoS API requests complete
44.7% of WoS API requests complete
44.9% of WoS API requests complete
45.0% of WoS API requests complete
45.2% of WoS API requests complete
45.4% of WoS API requests complete
45.5% of WoS API requests complete
45.7% of WoS API requests complete
45.9% of WoS API requests complete
46.1% of WoS API requests complete
46.2% of WoS API requests complete
46.4% of WoS API requests complete
46.6% of WoS API requests complete
46.7% of WoS API requests complete
46.9% of WoS API requests complete
47.1% of WoS API requests complete
47.2% of WoS API requests complete
47.4% of WoS API requests complete
47.6% of WoS API requests complete
47.7% of WoS API requests complete
47.9% of WoS API requests complete
48.1% of WoS API requests complete
48.2% of WoS API requests complete
48.4% of WoS API requests complete
48.6% of WoS API requests complete
48.7% of WoS API requests complete
48.9% of WoS API requests complete
49.1% of WoS API requests complete
49.2% of WoS API requests complete
49.4% of WoS API requests complete
49.6% of WoS API requests complete
49.7% of WoS API requests complete
49.9% of WoS API requests complete
50.1% of WoS API requests complete
50.3% of WoS API requests complete
50.4% of WoS API requests complete
50.6% of WoS API requests complete
50.8% of WoS API requests complete
50.9% of WoS API requests complete
51.1% of WoS API requests complete
51.3% of WoS API requests complete
51.4% of WoS API requests complete
51.6% of WoS API requests complete
51.8% of WoS API requests complete
51.9% of WoS API requests complete
52.1% of WoS API requests complete
52.3% of WoS API requests complete
52.4% of WoS API requests complete
52.6% of WoS API requests complete
52.8% of WoS API requests complete
52.9% of WoS API requests complete
53.1% of WoS API requests complete
53.3% of WoS API requests complete
53.4% of WoS API requests complete
53.6% of WoS API requests complete
53.8% of WoS API requests complete
53.9% of WoS API requests complete
54.1% of WoS API requests complete
54.3% of WoS API requests complete
54.5% of WoS API requests complete
54.6% of WoS API requests complete
54.8% of WoS API requests complete
55.0% of WoS API requests complete
55.1% of WoS API requests complete
55.3% of WoS API requests complete
55.5% of WoS API requests complete
55.6% of WoS API requests complete
55.8% of WoS API requests complete
56.0% of WoS API requests complete
56.1% of WoS API requests complete
56.3% of WoS API requests complete
56.5% of WoS API requests complete
56.6% of WoS API requests complete
56.8% of WoS API requests complete
57.0% of WoS API requests complete
57.1% of WoS API requests complete
57.3% of WoS API requests complete
57.5% of WoS API requests complete
57.6% of WoS API requests complete
57.8% of WoS API requests complete
58.0% of WoS API requests complete
58.2% of WoS API requests complete
58.3% of WoS API requests complete
58.5% of WoS API requests complete
58.7% of WoS API requests complete
58.8% of WoS API requests complete
59.0% of WoS API requests complete
59.2% of WoS API requests complete
59.3% of WoS API requests complete
59.5% of WoS API requests complete
59.7% of WoS API requests complete
59.8% of WoS API requests complete
60.0% of WoS API requests complete
60.2% of WoS API requests complete
60.3% of WoS API requests complete
60.5% of WoS API requests complete
60.7% of WoS API requests complete
60.8% of WoS API requests complete
61.0% of WoS API requests complete
61.2% of WoS API requests complete
61.3% of WoS API requests complete
61.5% of WoS API requests complete
61.7% of WoS API requests complete
61.8% of WoS API requests complete
62.0% of WoS API requests complete
62.2% of WoS API requests complete
62.4% of WoS API requests complete
62.5% of WoS API requests complete
62.7% of WoS API requests complete
62.9% of WoS API requests complete
63.0% of WoS API requests complete
63.2% of WoS API requests complete
63.4% of WoS API requests complete
63.5% of WoS API requests complete
63.7% of WoS API requests complete
63.9% of WoS API requests complete
64.0% of WoS API requests complete
64.2% of WoS API requests complete
64.4% of WoS API requests complete
64.5% of WoS API requests complete
64.7% of WoS API requests complete
64.9% of WoS API requests complete
65.0% of WoS API requests complete
65.2% of WoS API requests complete
65.4% of WoS API requests complete
65.5% of WoS API requests complete
65.7% of WoS API requests complete
65.9% of WoS API requests complete
66.1% of WoS API requests complete
66.2% of WoS API requests complete
66.4% of WoS API requests complete
66.6% of WoS API requests complete
66.7% of WoS API requests complete
66.9% of WoS API requests complete
67.1% of WoS API requests complete
67.2% of WoS API requests complete
67.4% of WoS API requests complete
67.6% of WoS API requests complete
67.7% of WoS API requests complete
67.9% of WoS API requests complete
68.1% of WoS API requests complete
68.2% of WoS API requests complete
68.4% of WoS API requests complete
68.6% of WoS API requests complete
68.7% of WoS API requests complete
68.9% of WoS API requests complete
69.1% of WoS API requests complete
69.2% of WoS API requests complete
69.4% of WoS API requests complete
69.6% of WoS API requests complete
69.7% of WoS API requests complete
69.9% of WoS API requests complete
70.1% of WoS API requests complete
70.3% of WoS API requests complete
70.4% of WoS API requests complete
70.6% of WoS API requests complete
70.8% of WoS API requests complete
70.9% of WoS API requests complete
71.1% of WoS API requests complete
71.3% of WoS API requests complete
71.4% of WoS API requests complete
71.6% of WoS API requests complete
71.8% of WoS API requests complete
71.9% of WoS API requests complete
72.1% of WoS API requests complete
72.3% of WoS API requests complete
72.4% of WoS API requests complete
72.6% of WoS API requests complete
72.8% of WoS API requests complete
72.9% of WoS API requests complete
73.1% of WoS API requests complete
73.3% of WoS API requests complete
73.4% of WoS API requests complete
73.6% of WoS API requests complete
73.8% of WoS API requests complete
73.9% of WoS API requests complete
74.1% of WoS API requests complete
74.3% of WoS API requests complete
74.5% of WoS API requests complete
74.6% of WoS API requests complete
74.8% of WoS API requests complete
75.0% of WoS API requests complete
75.1% of WoS API requests complete
75.3% of WoS API requests complete
75.5% of WoS API requests complete
75.6% of WoS API requests complete
75.8% of WoS API requests complete
76.0% of WoS API requests complete
76.1% of WoS API requests complete
76.3% of WoS API requests complete
76.5% of WoS API requests complete
76.6% of WoS API requests complete
76.8% of WoS API requests complete
77.0% of WoS API requests complete
77.1% of WoS API requests complete
77.3% of WoS API requests complete
77.5% of WoS API requests complete
77.6% of WoS API requests complete
77.8% of WoS API requests complete
78.0% of WoS API requests complete
78.2% of WoS API requests complete
78.3% of WoS API requests complete
78.5% of WoS API requests complete
78.7% of WoS API requests complete
78.8% of WoS API requests complete
79.0% of WoS API requests complete
79.2% of WoS API requests complete
79.3% of WoS API requests complete
79.5% of WoS API requests complete
79.7% of WoS API requests complete
79.8% of WoS API requests complete
80.0% of WoS API requests complete
80.2% of WoS API requests complete
80.3% of WoS API requests complete
80.5% of WoS API requests complete
80.7% of WoS API requests complete
80.8% of WoS API requests complete
81.0% of WoS API requests complete
81.2% of WoS API requests complete
81.3% of WoS API requests complete
81.5% of WoS API requests complete
81.7% of WoS API requests complete
81.8% of WoS API requests complete
82.0% of WoS API requests complete
82.2% of WoS API requests complete
82.4% of WoS API requests complete
82.5% of WoS API requests complete
82.7% of WoS API requests complete
82.9% of WoS API requests complete
83.0% of WoS API requests complete
83.2% of WoS API requests complete
83.4% of WoS API requests complete
83.5% of WoS API requests complete
83.7% of WoS API requests complete
83.9% of WoS API requests complete
84.0% of WoS API requests complete
84.2% of WoS API requests complete
84.4% of WoS API requests complete
84.5% of WoS API requests complete
84.7% of WoS API requests complete
84.9% of WoS API requests complete
85.0% of WoS API requests complete
85.2% of WoS API requests complete
85.4% of WoS API requests complete
85.5% of WoS API requests complete
85.7% of WoS API requests complete
85.9% of WoS API requests complete
86.1% of WoS API requests complete
86.2% of WoS API requests complete
86.4% of WoS API requests complete
86.6% of WoS API requests complete
86.7% of WoS API requests complete
86.9% of WoS API requests complete
87.1% of WoS API requests complete
87.2% of WoS API requests complete
87.4% of WoS API requests complete
87.6% of WoS API requests complete
87.7% of WoS API requests complete
87.9% of WoS API requests complete
88.1% of WoS API requests complete
88.2% of WoS API requests complete
88.4% of WoS API requests complete
88.6% of WoS API requests complete
88.7% of WoS API requests complete
88.9% of WoS API requests complete
89.1% of WoS API requests complete
89.2% of WoS API requests complete
89.4% of WoS API requests complete
89.6% of WoS API requests complete
89.7% of WoS API requests complete
89.9% of WoS API requests complete
90.1% of WoS API requests complete
90.3% of WoS API requests complete
90.4% of WoS API requests complete
90.6% of WoS API requests complete
90.8% of WoS API requests complete
90.9% of WoS API requests complete
91.1% of WoS API requests complete
91.3% of WoS API requests complete
91.4% of WoS API requests complete
91.6% of WoS API requests complete
91.8% of WoS API requests complete
91.9% of WoS API requests complete
92.1% of WoS API requests complete
92.3% of WoS API requests complete
92.4% of WoS API requests complete
92.6% of WoS API requests complete
92.8% of WoS API requests complete
92.9% of WoS API requests complete
93.1% of WoS API requests complete
93.3% of WoS API requests complete
93.4% of WoS API requests complete
93.6% of WoS API requests complete
93.8% of WoS API requests complete
93.9% of WoS API requests complete
94.1% of WoS API requests complete
94.3% of WoS API requests complete
94.5% of WoS API requests complete
94.6% of WoS API requests complete
94.8% of WoS API requests complete
95.0% of WoS API requests complete
95.1% of WoS API requests complete
95.3% of WoS API requests complete
95.5% of WoS API requests complete
95.6% of WoS API requests complete
95.8% of WoS API requests complete
96.0% of WoS API requests complete
96.1% of WoS API requests complete
96.3% of WoS API requests complete
96.5% of WoS API requests complete
96.6% of WoS API requests complete
96.8% of WoS API requests complete
97.0% of WoS API requests complete
97.1% of WoS API requests complete
97.3% of WoS API requests complete
97.5% of WoS API requests complete
97.6% of WoS API requests complete
97.8% of WoS API requests complete
98.0% of WoS API requests complete
98.2% of WoS API requests complete
98.3% of WoS API requests complete
98.5% of WoS API requests complete
98.7% of WoS API requests complete
98.8% of WoS API requests complete
99.0% of WoS API requests complete
99.2% of WoS API requests complete
99.3% of WoS API requests complete
99.5% of WoS API requests complete
99.7% of WoS API requests complete
99.8% of WoS API requests complete
100.0% of WoS API requests complete
100.2% of WoS API requests complete

We obtain a list of dictionaries including all the records extracted (as an example, the data for the first paper is shown below). We can save these raw results in a .json file.

wos_data[0]
{'uid': 'WOS:000187876000038',
 'title': 'A critical study of the thermoelectric method of measuring vapor pressure',
 'types': ['Article'],
 'sourceTypes': ['Article'],
 'source': {'sourceTitle': 'JOURNAL OF BIOLOGICAL CHEMISTRY',
  'publishYear': 1938,
  'publishMonth': 'NOV',
  'volume': '126',
  'issue': '1',
  'pages': {'range': '349-360', 'begin': '349', 'end': '360', 'count': 12}},
 'names': {'authors': [{'displayName': 'Roepke, RR',
    'wosStandard': 'Roepke, RR',
    'researcherId': 'FXG-1059-2022'},
   {'displayName': 'Baldes, EJ',
    'wosStandard': 'Baldes, EJ',
    'researcherId': 'EKH-9490-2022'}]},
 'links': {'record': 'https://www.webofscience.com/api/gateway?GWVersion=2&SrcApp=incarcsic_woslite&SrcAuth=WosAPI&KeyUT=WOS:000187876000038&DestLinkType=FullRecord&DestApp=WOS_CPL',
  'citingArticles': 'https://www.webofscience.com/api/gateway?GWVersion=2&SrcApp=incarcsic_woslite&SrcAuth=WosAPI&KeyUT=WOS:000187876000038&DestLinkType=CitingArticles&DestApp=WOS_CPL',
  'references': 'https://www.webofscience.com/api/gateway?GWVersion=2&SrcApp=incarcsic_woslite&SrcAuth=WosAPI&KeyUT=WOS:000187876000038&DestLinkType=CitedReferences&DestApp=WOS',
  'related': 'https://www.webofscience.com/api/gateway?GWVersion=2&SrcApp=incarcsic_woslite&SrcAuth=WosAPI&KeyUT=WOS:000187876000038&DestLinkType=RelatedRecords&DestApp=WOS_CPL'},
 'citations': [{'db': 'WOS', 'count': 10}],
 'identifiers': {'eissn': '1083-351X'},
 'keywords': {'authorKeywords': []}}
with open("wos_data_output.json", "w") as file:
    json.dump(wos_data, file)
with open("wos_data_output.json") as file:
    wos_data = json.load(file)
len(wos_data)
29704

We process the obtained data with the function process_data and save the results in a .csv file with the function save_results.

years = process_data(wos_data)
save_results("thermoelectric", years)

papers = 0
for year in years:
    papers += year["wos_documents"]
print(
    f"Number of papers published between 1996 and 2023 from the query ({query}): {papers}"
)
Number of papers published between 1996 and 2023 from the query (thermoelectric AND materials): 22867

Following this procedure, we have extracted from WoS the data on published papers (Articles and Data Papers) for the main categories of materials studied in chemistry. Using the corresponding APIs, we can also extract the number of datasets published in the DataCite and ZENODO repositories for the same categories.

9.2. Collecting the saved data for all categories from WoS, DataCite and ZENODO#

In this section, we prepare the collected data on papers and datasets to plot the results as a function of the categories and publication years.

from glob import glob
all_results = glob("./wos_results/*.csv")
print(f"files found: {len(all_results)}")
files found: 9
all_results
['./wos_results/nanoparticles.csv',
 './wos_results/polymers.csv',
 './wos_results/mofs.csv',
 './wos_results/photocatalytic.csv',
 './wos_results/biomaterials.csv',
 './wos_results/thermoelectric.csv',
 './wos_results/battery_cathode.csv',
 './wos_results/2Dmaterials.csv',
 './wos_results/semiconductor.csv']

We create a .csv file combining the results of the different categories.

df = pd.read_csv(all_results[0], sep=";")
df.head()
publication_year wos_documents
0 2023 76010
1 2022 79936
2 2021 73154
3 2020 75213
4 2019 70984
df = df.sort_values(by="publication_year").reset_index(drop=True)
df_results = pd.DataFrame(df["publication_year"].reset_index(drop=True)).copy()
df_results.head()
publication_year
0 1996
1 1997
2 1998
3 1999
4 2000
materials_name = [
    "nanoparticles",
    "battery_cathode",
    "photocatalytic",
    "polymers",
    "thermoelectric",
    "mofs",
    "biomaterials",
    "2Dmaterials",
    "semiconductor",
]
for file in materials_name:
    df = pd.read_csv(f"./wos_results/{file}.csv", sep=";")
    df = df.sort_values(by="publication_year").reset_index(drop=True)
    df = df.rename(columns={"wos_documents": file})
    df_results = pd.merge(df_results, df)
df_results.head()
publication_year nanoparticles battery_cathode photocatalytic polymers thermoelectric mofs biomaterials 2Dmaterials semiconductor
0 1996 312 49 15 12406 142 30 233 121 570
1 1997 454 56 18 12661 146 44 240 137 654
2 1998 749 97 25 13879 212 37 294 164 695
3 1999 1022 121 28 14713 226 47 347 178 721
4 2000 1298 160 27 15088 294 45 345 188 666
df_results.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 28 entries, 0 to 27
Data columns (total 10 columns):
 #   Column            Non-Null Count  Dtype
---  ------            --------------  -----
 0   publication_year  28 non-null     int64
 1   nanoparticles     28 non-null     int64
 2   battery_cathode   28 non-null     int64
 3   photocatalytic    28 non-null     int64
 4   polymers          28 non-null     int64
 5   thermoelectric    28 non-null     int64
 6   mofs              28 non-null     int64
 7   biomaterials      28 non-null     int64
 8   2Dmaterials       28 non-null     int64
 9   semiconductor     28 non-null     int64
dtypes: int64(10)
memory usage: 2.3 KB
df_results.to_csv("wos_materials_papers.csv", sep=",", index=False)

9.3. Plot the number of research papers vs. datasets deposited in data repositories in chemistry and materials science per year#

9.3.1. Load data on papers from Web of Science#

df_wos = pd.read_csv("wos_materials_papers.csv", sep=",")
df_wos.head()
publication_year nanoparticles battery_cathode photocatalytic polymers thermoelectric mofs biomaterials 2Dmaterials semiconductor
0 1996 312 49 15 12406 142 30 233 121 570
1 1997 454 56 18 12661 146 44 240 137 654
2 1998 749 97 25 13879 212 37 294 164 695
3 1999 1022 121 28 14713 226 47 347 178 721
4 2000 1298 160 27 15088 294 45 345 188 666

9.3.2. Load data on DataCite datasets#

df_datacite = pd.read_csv("datacite_datasets.csv", sep=";")
df_datacite.head()
publication_year nanoparticles battery_cathode photocatalytic polymers thermoelectric mofs biomaterials 2Dmaterials semiconductor
0 2010 17 NaN NaN 11 NaN 1 1 2 NaN
1 2011 36 NaN NaN 4 1.0 1 2 4 NaN
2 2012 30 NaN NaN 41 NaN 1 2 4 NaN
3 2013 52 NaN NaN 12 1.0 1 2 139 1.0
4 2014 148 1.0 1.0 53 1.0 3 8 33 1.0

9.3.3. Load data on ZENODO datasets#

df_zenodo = pd.read_csv("zenodo_datasets.csv", sep=";")
df_zenodo.head()
publication_year nanoparticles battery_cathode photocatalytic polymers thermoelectric mofs biomaterials 2Dmaterials semiconductor
0 2010 NaN NaN NaN NaN NaN NaN 1.0 NaN NaN
1 2011 NaN NaN NaN NaN NaN NaN NaN NaN NaN
2 2012 2.0 NaN NaN 1.0 NaN NaN NaN NaN NaN
3 2013 NaN NaN NaN NaN NaN NaN NaN NaN NaN
4 2014 NaN NaN NaN 2.0 NaN 1.0 NaN NaN NaN

9.3.4. Unify labels and colors#

labels = [
    "nanoparticles",
    "battery cathode materials",
    "photocatalytic materials",
    "polymers",
    "thermoelectric materials",
    "MOFs",
    "biomaterials",
    "2D materials",
    "semiconductor materials",
]
colors = [
    "#1a4a81",
    "#2166ac",
    "#4393c3",
    "#92c5de",
    "#c2e1f2",
    "#f4a582",
    "#d6604d",
    "#b2182b",
    "#67001f",
]

9.3.5. Plot WoS chemistry and materials science papers vs. year using plotly.express#

import plotly.express as px
fig = px.bar(
    data_frame=df_wos,
    x="publication_year",
    y=df_wos.columns[1:],
    barmode="relative",
    color_discrete_sequence=colors,
)

for i, trace in enumerate(fig.data):
    trace.update(name=labels[i])

fig.update_layout(
    {"plot_bgcolor": "#FFFFFF", "paper_bgcolor": "#FFFFFF"},
    width=900,
    height=500,
    margin=dict(l=100, r=40, t=20, b=45),
    font_family="Arial",
    font_color="#646363",
    font_size=20,
    title_font_family="Arial",
    title_font_color="#646363",
    legend_title_text=None,
    legend=dict(font={"size": 18}, yanchor="bottom", y=0.44, xanchor="left", x=0.04),
)

fig.update_yaxes(
    title_text="no. WoS papers",
    showgrid=True,
    gridcolor="#9D9D9C",
    title={"font": {"size": 24}},
    range=(0, 220000),
    tickformat=".2s",
)
fig.update_xaxes(
    title_text=None,
    linecolor="#9D9D9C",
    range=(1995.5, 2023.5),
    tickvals=[1996, 1999, 2002, 2005, 2008, 2011, 2014, 2017, 2020, 2023],
)

fig