Skip to content

Commit 9b0ffb1

Browse files
committed
Replace sub region with population, add diff and flags
1 parent be0954c commit 9b0ffb1

File tree

7 files changed

+170
-120
lines changed

7 files changed

+170
-120
lines changed

examples/example3/README.md

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
This is a simple example to show how DiffSync can be used to compare and synchronize data with a remote system like via a REST API like Nautobot.
55

66
For this example, we have a shared model for Region and Country defined in `models.py`.
7-
A Country must be associated with a Region and can be part of a Subregion too.
7+
A country must be part of a region and has an attribute to capture its population.
88

99
The comparison and synchronization of dataset is done between a local JSON file and the [public instance of Nautobot](https://demo.nautobot.com).
1010

@@ -16,17 +16,26 @@ to use this example you must have some dependencies installed, please make sure
1616
pip install -r requirements.txt
1717
```
1818

19+
## Setup the environment
20+
21+
By default this example will interact with the public sandbox of Nautobot at https://demo.nautobot.com but you can use your own version of Nautobot by providing a new URL and a new API token using the environment variables `NAUTOBOT_URL` & `NAUTOBOT_TOKEN`
22+
23+
```
24+
export NAUTOBOT_URL = "https://demo.nautobot.com"
25+
export NAUTOBOT_TOKEN = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
26+
```
27+
1928
## Try the example
2029

2130
The first time a lot of changes should be reported between Nautobot and the local data because by default the demo instance doesn't have the subregion define.
2231
After the first sync, the diff should show no difference.
23-
At this point, Diffsync will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode.
32+
At this point, `Diffsync` will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode.
2433

2534
```
2635
### DIFF Compare the data between Nautobot and the local JSON file.
27-
main.py --diff
36+
python main.py --diff
2837
2938
### SYNC Update the list of country in Nautobot.
30-
main.py --sync
39+
python main.py --sync
3140
```
3241

examples/example3/diff.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from diffsync.diff import Diff
2+
3+
4+
class AlphabeticalOrderDiff(Diff):
5+
"""Simple diff to return all children country in alphabetical order."""
6+
7+
@classmethod
8+
def order_children_default(cls, children):
9+
"""Simple diff to return all children in alphabetical order."""
10+
for child_name, child in sorted(children.items()):
11+
12+
# it's possible to access additional information about the object
13+
# like child.action can be "update", "create" or "delete"
14+
15+
yield children[child_name]

examples/example3/local_adapter.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,11 @@ def load(self, filename=COUNTRIES_FILE):
4545
region = self.get(obj=self.region, identifier=slugify(country.get("region")))
4646

4747
name = country.get("country")
48-
item = self.country(
49-
slug=slugify(name), name=name, subregion=country.get("subregion", None), region=region.slug
50-
)
48+
49+
# The population is store in thousands in the local file so we need to convert it
50+
population = int(float(country.get("pop2021")) * 1000)
51+
52+
item = self.country(slug=slugify(name), name=name, population=population, region=region.slug)
5153
self.add(item)
5254

5355
region.add_child(item)

examples/example3/main.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
import pprint
66

77
from diffsync import Diff
8+
from diffsync.enum import DiffSyncFlags
89
from diffsync.logging import enable_console_logging
910

1011
from local_adapter import LocalAdapter
1112
from nautobot_adapter import NautobotAdapter
13+
from diff import AlphabeticalOrderDiff
1214

1315

1416
def main():
@@ -26,21 +28,21 @@ def main():
2628
print("Initializing and loading Local Data ...")
2729
local = LocalAdapter()
2830
local.load()
29-
# print(local.str())
3031

3132
print("Initializing and loading Nautobot Data ...")
3233
nautobot = NautobotAdapter()
3334
nautobot.load()
34-
# print(nautobot.str())
35+
36+
flags = DiffSyncFlags.SKIP_UNMATCHED_DST
3537

3638
if args.diff:
3739
print("Calculating the Diff between the local adapter and Nautobot ...")
38-
diff = nautobot.diff_from(local)
40+
diff = nautobot.diff_from(local, flags=flags, diff_class=AlphabeticalOrderDiff)
3941
print(diff.str())
4042

4143
elif args.sync:
4244
print("Updating the list of countries in Nautobot ...")
43-
nautobot.sync_from(local)
45+
nautobot.sync_from(local, flags=flags, diff_class=AlphabeticalOrderDiff)
4446

4547

4648
if __name__ == "__main__":

examples/example3/models.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ class Region(DiffSyncModel):
2121
class Country(DiffSyncModel):
2222
"""Example model of a Country.
2323
24-
A must be part of a region and can be also associated with a subregion.
24+
A country must be part of a region and has an attribute to capture its population.
2525
"""
2626

2727
_modelname = "country"
2828
_identifiers = ("slug",)
29-
_attributes = ("name", "region", "subregion")
29+
_attributes = ("name", "region", "population")
3030

3131
slug: str
3232
name: str
3333
region: str
34-
subregion: Optional[str]
34+
population: Optional[int]

examples/example3/nautobot_adapter.py

Lines changed: 24 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -2,110 +2,21 @@
22
import pynautobot
33

44
from diffsync import DiffSync
5-
from models import Region, Country
5+
6+
from nautobot_models import NautobotCountry, NautobotRegion
67

78
NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com")
89
NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
910

10-
11-
class NautobotRegion(Region):
12-
"""Extend the Region object to store Nautobot specific information.
13-
14-
Region are represented in Nautobot as a dcim.region object without parent.
15-
"""
16-
17-
remote_id: str
18-
"""Store the nautobot uuid in the object to allow update and delete of existing object."""
19-
20-
21-
class NautobotCountry(Country):
22-
"""Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE.
23-
24-
Country are represented in Nautobot as a dcim.region object as well but a country must have a parent.
25-
Subregion information will be store in the description of the object in Nautobot
26-
"""
27-
28-
remote_id: str
29-
"""Store the nautobot uuid in the object to allow update and delete of existing object."""
30-
31-
@classmethod
32-
def create(cls, diffsync: "DiffSync", ids: dict, attrs: dict):
33-
"""Create a country object in Nautobot.
34-
35-
Args:
36-
diffsync: The master data store for other DiffSyncModel instances that we might need to reference
37-
ids: Dictionary of unique-identifiers needed to create the new object
38-
attrs: Dictionary of additional attributes to set on the new object
39-
40-
Returns:
41-
NautobotCountry: DiffSync object newly created
42-
"""
43-
44-
# Retrieve the parent region in internal cache to access its UUID
45-
# because the UUID is required to associate the object to its parent region in Nautobot
46-
region = diffsync.get(diffsync.region, attrs.get("region"))
47-
48-
# Create the new country in Nautobot and attach it to its parent
49-
try:
50-
country = diffsync.nautobot.dcim.regions.create(
51-
slug=ids.get("slug"),
52-
name=attrs.get("name"),
53-
description=attrs.get("subregion", None),
54-
parent=region.remote_id,
55-
)
56-
print(f"Created country : {ids} | {attrs} | {country.id}")
57-
58-
except pynautobot.core.query.RequestError as exc:
59-
print(f"Unable to create country {ids} | {attrs} | {exc}")
60-
return None
61-
62-
# Add the newly created remote_id and create the internal object for this resource.
63-
attrs["remote_id"] = country.id
64-
item = super().create(ids=ids, diffsync=diffsync, attrs=attrs)
65-
return item
66-
67-
def update(self, attrs: dict):
68-
"""Update a country object in Nautobot.
69-
70-
Args:
71-
attrs: Dictionary of attributes to update on the object
72-
73-
Returns:
74-
DiffSyncModel: this instance, if all data was successfully updated.
75-
None: if data updates failed in such a way that child objects of this model should not be modified.
76-
77-
Raises:
78-
ObjectNotUpdated: if an error occurred.
79-
"""
80-
81-
# Retrive the pynautobot object from Nautobot since we only have the UUID internally
82-
remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
83-
84-
# Convert the internal attrs to Nautobot format
85-
nautobot_attrs = {}
86-
if "subregion" in attrs:
87-
nautobot_attrs["description"] = attrs.get("subregion")
88-
if "name" in attrs:
89-
nautobot_attrs["name"] = attrs.get("name")
90-
91-
if nautobot_attrs:
92-
remote.update(data=nautobot_attrs)
93-
print(f"Updated Country {self.slug} | {attrs}")
94-
95-
return super().update(attrs)
96-
97-
def delete(self):
98-
"""Delete a country object in Nautobot.
99-
100-
Returns:
101-
NautobotCountry: DiffSync object
102-
"""
103-
# Retrieve the pynautobot object and delete the object in Nautobot
104-
remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
105-
remote.delete()
106-
107-
super().delete()
108-
return self
11+
CUSTOM_FIELDS = [
12+
{
13+
"name": "country_population",
14+
"display": "Population (nbr people)",
15+
"content_types": ["dcim.region"],
16+
"type": "integer",
17+
"description": "Number of inhabitant per country",
18+
}
19+
]
10920

11021

11122
class NautobotAdapter(DiffSync):
@@ -132,7 +43,7 @@ def load(self):
13243

13344
# Initialize pynautobot to interact with Nautobot and store it within the adapter
13445
# to reuse it later
135-
self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN,)
46+
self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN)
13647

13748
# Pull all regions from Nautobot, which includes all regions and all countries
13849
regions = self.nautobot.dcim.regions.all()
@@ -142,10 +53,6 @@ def load(self):
14253
if region.parent:
14354
continue
14455

145-
# We are excluding the networktocode because it's not present in the local file
146-
if region.slug == "networktocode":
147-
continue
148-
14956
item = self.region(slug=region.slug, name=region.name, remote_id=region.id)
15057
self.add(item)
15158

@@ -160,8 +67,19 @@ def load(self):
16067
slug=country.slug,
16168
name=country.name,
16269
region=parent.slug,
163-
subregion=country.description,
70+
population=country.custom_fields.get("country_population", None),
16471
remote_id=country.id,
16572
)
16673
self.add(item)
16774
parent.add_child(item)
75+
76+
def sync_from(self, *args, **kwargs):
77+
"""Sync the data with Nautobot but first ensure that all the required Custom fields are present in Nautobot."""
78+
79+
# Check if all required custom fields exist, create them if they don't
80+
for custom_field in CUSTOM_FIELDS:
81+
nb_cfs = self.cfs = self.nautobot.extras.custom_fields.filter(name=custom_field.get("name"))
82+
if not nb_cfs:
83+
self.nautobot.extras.custom_fields.create(**custom_field)
84+
85+
super().sync_from(*args, **kwargs)

examples/example3/nautobot_models.py

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import os
2+
import pynautobot
3+
4+
5+
from diffsync import DiffSync
6+
from models import Region, Country
7+
8+
9+
class NautobotRegion(Region):
10+
"""Extend the Region object to store Nautobot specific information.
11+
12+
Region are represented in Nautobot as a dcim.region object without parent.
13+
"""
14+
15+
remote_id: str
16+
"""Store the nautobot uuid in the object to allow update and delete of existing object."""
17+
18+
19+
class NautobotCountry(Country):
20+
"""Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE.
21+
22+
Country are represented in Nautobot as a dcim.region object as well but a country must have a parent.
23+
Subregion information will be store in the description of the object in Nautobot
24+
"""
25+
26+
remote_id: str
27+
"""Store the nautobot uuid in the object to allow update and delete of existing object."""
28+
29+
@classmethod
30+
def create(cls, diffsync: DiffSync, ids: dict, attrs: dict):
31+
"""Create a country object in Nautobot.
32+
33+
Args:
34+
diffsync: The master data store for other DiffSyncModel instances that we might need to reference
35+
ids: Dictionary of unique-identifiers needed to create the new object
36+
attrs: Dictionary of additional attributes to set on the new object
37+
38+
Returns:
39+
NautobotCountry: DiffSync object newly created
40+
"""
41+
42+
# Retrieve the parent region in internal cache to access its UUID
43+
# because the UUID is required to associate the object to its parent region in Nautobot
44+
region = diffsync.get(diffsync.region, attrs.get("region"))
45+
46+
# Create the new country in Nautobot and attach it to its parent
47+
try:
48+
country = diffsync.nautobot.dcim.regions.create(
49+
slug=ids.get("slug"),
50+
name=attrs.get("name"),
51+
custom_fields=dict(population=attrs.get("population")),
52+
parent=region.remote_id,
53+
)
54+
print(f"Created country : {ids} | {attrs} | {country.id}")
55+
56+
except pynautobot.core.query.RequestError as exc:
57+
print(f"Unable to create country {ids} | {attrs} | {exc}")
58+
return None
59+
60+
# Add the newly created remote_id and create the internal object for this resource.
61+
attrs["remote_id"] = country.id
62+
item = super().create(ids=ids, diffsync=diffsync, attrs=attrs)
63+
return item
64+
65+
def update(self, attrs: dict):
66+
"""Update a country object in Nautobot.
67+
68+
Args:
69+
attrs: Dictionary of attributes to update on the object
70+
71+
Returns:
72+
DiffSyncModel: this instance, if all data was successfully updated.
73+
None: if data updates failed in such a way that child objects of this model should not be modified.
74+
75+
Raises:
76+
ObjectNotUpdated: if an error occurred.
77+
"""
78+
79+
# Retrive the pynautobot object from Nautobot since we only have the UUID internally
80+
remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
81+
82+
# Convert the internal attrs to Nautobot format
83+
if "population" in attrs:
84+
remote.custom_fields["country_population"] = attrs.get("population")
85+
if "name" in attrs:
86+
remote.name = attrs.get("name")
87+
88+
remote.save()
89+
print(f"Updated Country {self.slug} | {attrs}")
90+
91+
return super().update(attrs)
92+
93+
def delete(self):
94+
"""Delete a country object in Nautobot.
95+
96+
Returns:
97+
NautobotCountry: DiffSync object
98+
"""
99+
# Retrieve the pynautobot object and delete the object in Nautobot
100+
remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
101+
remote.delete()
102+
103+
super().delete()
104+
return self

0 commit comments

Comments
 (0)