(async () => {
const web = _spPageContextInfo.webAbsoluteUrl;
const EMPTY_LISTS_FIRST = false;
const SKIP_EXISTING_ITEMS = true;
// ---- load JSON via file picker ----
const data = await new Promise((resolve, reject) => {
const input = document.createElement("input");
input.type = "file";
input.accept = "application/json,.json";
input.onchange = () => {
const file = input.files[0];
if (!file) return reject(new Error("No file selected"));
const reader = new FileReader();
reader.onload = () => { try { resolve(JSON.parse(reader.result)); } catch (e) { reject(e); } };
reader.onerror = () => reject(reader.error);
reader.readAsText(file);
};
input.click();
});
console.log(`[Import] Loaded export from ${data.source?.web} with ${data.lists.length} lists`);
// ---- REST helpers ----
const digest = (await (await fetch(`${web}/_api/contextinfo`, {
method: "POST", headers: { Accept: "application/json;odata=verbose" },
})).json()).d.GetContextWebInformation.FormDigestValue;
const getJson = async (url) => {
const r = await fetch(url, { headers: { Accept: "application/json;odata=verbose" } });
if (!r.ok) return { ok: false, status: r.status, body: await r.text() };
return { ok: true, d: (await r.json()).d };
};
const send = async (url, body, method = "POST", extraHeaders = {}) => {
const r = await fetch(url, {
method,
headers: {
Accept: "application/json;odata=verbose",
"Content-Type": "application/json;odata=verbose",
"X-RequestDigest": digest,
...extraHeaders,
},
body: JSON.stringify(body),
});
if (!r.ok) throw new Error(`${method} ${url} -> ${r.status}: ${await r.text()}`);
const t = await r.text();
return t ? JSON.parse(t).d : null;
};
const esc = (s) => String(s).replace(/'/g, "''");
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// Delete every item in a list (used when EMPTY_LISTS_FIRST is true).
const clearListItems = async (title) => {
const ids = [];
let url = `${web}/_api/web/lists/getbytitle('${esc(title)}')/items?$select=Id&$top=500`;
while (url) {
const r = await getJson(url);
if (!r.ok) break;
ids.push(...r.d.results.map((x) => x.Id));
url = r.d.__next || null;
}
for (const id of ids) {
try {
await send(
`${web}/_api/web/lists/getbytitle('${esc(title)}')/items(${id})`,
{}, "POST", { "X-HTTP-Method": "DELETE", "IF-MATCH": "*" }
);
} catch (e) {
console.warn(`[Import] ! delete item #${id} in ${title} failed: ${e.message}`);
}
}
console.log(`[Import] cleared ${ids.length} existing items from ${title}`);
};
const listByTitle = {}; // title -> { id, entityType }
const idMap = {}; // listTitle -> { sourceItemId -> newItemId }
const userCache = {}; // email -> id (or null)
const ensureUser = async (email) => {
if (!email) return null;
if (email in userCache) return userCache[email];
try {
const d = await send(`${web}/_api/web/ensureuser`, { logonName: email });
userCache[email] = d.Id;
} catch (e) {
console.warn(`[Import] ensureUser failed for ${email}: ${e.message}`);
userCache[email] = null;
}
return userCache[email];
};
const getListInfo = async (title) => {
if (listByTitle[title]) return listByTitle[title];
const r = await getJson(`${web}/_api/web/lists/getbytitle('${esc(title)}')?$select=Id,ListItemEntityTypeFullName`);
if (!r.ok) return null;
listByTitle[title] = { id: r.d.Id, entityType: r.d.ListItemEntityTypeFullName };
return listByTitle[title];
};
// SharePoint list provisioning is eventually consistent: a getbytitle right
// after POST /lists can 404 for a second or two. Poll until the list is
// queryable so later passes (lookups, items) never miss a freshly made list.
const waitForList = async (title, tries = 10, delayMs = 800) => {
for (let i = 0; i < tries; i++) {
delete listByTitle[title];
const info = await getListInfo(title);
if (info) return info;
await sleep(delayMs);
}
return null;
};
const hasField = async (title, internalName) => {
const r = await getJson(`${web}/_api/web/lists/getbytitle('${esc(title)}')/fields/getbyinternalnameortitle('${esc(internalName)}')`);
return r.ok;
};
const addFieldXml = async (title, schemaXml) => {
// A field add right after the list is created can transiently fail while
// the list finishes provisioning; retry a couple of times before giving up.
let lastErr;
for (let i = 0; i < 3; i++) {
try {
await send(`${web}/_api/web/lists/getbytitle('${esc(title)}')/fields/createfieldasxml`, {
parameters: {
__metadata: { type: "SP.XmlSchemaFieldCreationInformation" },
SchemaXml: schemaXml,
Options: 8, // AddFieldInternalNameHint
},
});
return;
} catch (e) {
lastErr = e;
await sleep(600);
}
}
throw lastErr;
};
// ============ PASS 0: create EVERY list first ============
// Creating all lists up front (and waiting until each is queryable) means
// the lookup pass below can always resolve its target list, regardless of
// the order lists appear in the export file.
for (const list of data.lists) {
if (await getListInfo(list.title)) {
console.log(`[Import] = list exists: ${list.title}`);
continue;
}
console.log(`[Import] + create list: ${list.title}`);
try {
await send(`${web}/_api/web/lists`, {
__metadata: { type: "SP.List" },
BaseTemplate: list.baseTemplate || 100,
Title: list.title,
Description: list.description || "",
});
} catch (e) {
console.warn(`[Import] ! create list ${list.title} failed: ${e.message}`);
continue;
}
if (!(await waitForList(list.title))) {
console.warn(`[Import] ! list ${list.title} not queryable after retries (provisioning slow?)`);
}
}
// ============ PASS 1: non-lookup fields ============
for (const list of data.lists) {
if (!(await getListInfo(list.title))) {
console.warn(`[Import] ! skipping fields, list missing: ${list.title}`);
continue;
}
for (const f of list.fields) {
if (f.internalName === "Title" || f.unsupported || f.isLookup) continue; // lookups in pass 2
if (await hasField(list.title, f.internalName)) continue;
try {
await addFieldXml(list.title, f.schemaXml);
console.log(`[Import] + field ${list.title}.${f.internalName}`);
} catch (e) {
console.warn(`[Import] ! field ${list.title}.${f.internalName} failed: ${e.message}`);
}
}
}
// ============ PASS 2: lookup fields (re-pointed to target GUIDs) ============
const skippedLookups = [];
for (const list of data.lists) {
for (const f of list.fields.filter((x) => x.isLookup && !x.unsupported)) {
if (await hasField(list.title, f.internalName)) continue;
if (!f.lookupListTitle) { console.warn(`[Import] ! lookup ${list.title}.${f.internalName} has no target list, skipped`); skippedLookups.push(`${list.title}.${f.internalName}`); continue; }
// waitForList (not getListInfo) so a slow-provisioning target is retried.
const target = await waitForList(f.lookupListTitle);
if (!target) { console.warn(`[Import] ! lookup target '${f.lookupListTitle}' not found for ${list.title}.${f.internalName}, skipped`); skippedLookups.push(`${list.title}.${f.internalName} -> ${f.lookupListTitle}`); continue; }
const type = f.multi ? "LookupMulti" : "Lookup";
const mult = f.multi ? "Mult='TRUE'" : "";
// NOTE: not marked Required so the two-pass item import can run.
const xml =
`<Field Type='${type}' DisplayName='${f.title}' Name='${f.internalName}' ` +
`StaticName='${f.internalName}' List='{${target.id}}' ShowField='${f.showField}' ${mult} />`;
try {
await addFieldXml(list.title, xml);
console.log(`[Import] + lookup ${list.title}.${f.internalName} -> ${f.lookupListTitle}`);
} catch (e) {
console.warn(`[Import] ! lookup ${list.title}.${f.internalName} failed: ${e.message}`);
}
}
}
// ============ PASS 3: items (scalar/choice/url) ============
const isMultiChoice = (f) => f.type === "MultiChoice";
const isUrl = (f) => f.type === "URL";
// Fields that actually get written as item values (used for dedup signature).
const scalarFieldsOf = (list) =>
list.fields.filter((f) => !f.unsupported && !f.isLookup && !f.isUser && !f.readOnly);
// Normalize one field value to a stable string so a source item and an
// existing target item produce the SAME signature when they are equal.
const normVal = (f, v) => {
if (v === undefined || v === null) return "";
if (isMultiChoice(f)) {
const arr = Array.isArray(v) ? v : (v.results || []);
return [...arr].map(String).sort().join("|");
}
if (isUrl(f)) return String((v && v.Url) || "");
if (typeof v === "object") return JSON.stringify(v);
return String(v);
};
const signatureOf = (scalarFields, item) =>
scalarFields.map((f) => normVal(f, item[f.internalName])).join("\u00A7");
// Read every existing item in a target list and map signature -> existing Id.
const fetchExistingSignatures = async (list, scalarFields) => {
const map = new Map();
if (!scalarFields.length) return map;
const sel = ["Id", ...scalarFields.map((f) => f.internalName)].join(",");
let url = `${web}/_api/web/lists/getbytitle('${esc(list.title)}')/items?$select=${sel}&$top=500`;
while (url) {
const r = await getJson(url);
if (!r.ok) break;
for (const it of r.d.results) map.set(signatureOf(scalarFields, it), it.Id);
url = r.d.__next || null;
}
return map;
};
for (const list of data.lists) {
const info = await getListInfo(list.title);
idMap[list.title] = {};
if (EMPTY_LISTS_FIRST) await clearListItems(list.title);
const scalarFields = scalarFieldsOf(list);
const existing = SKIP_EXISTING_ITEMS
? await fetchExistingSignatures(list, scalarFields)
: new Map();
if (SKIP_EXISTING_ITEMS) {
console.log(`[Import] items -> ${list.title} (${list.items.length}); ${existing.size} already in target`);
} else {
console.log(`[Import] items -> ${list.title} (${list.items.length})`);
}
let added = 0, skipped = 0;
for (const item of list.items) {
// De-dup: if an identical item already exists, reuse its Id and skip insert.
if (SKIP_EXISTING_ITEMS) {
const sig = signatureOf(scalarFields, item);
if (existing.has(sig)) {
idMap[list.title][item.__id] = existing.get(sig);
skipped++;
continue;
}
}
const body = { __metadata: { type: info.entityType } };
for (const f of list.fields) {
if (f.unsupported || f.isLookup || f.isUser) continue;
if (f.readOnly) continue;
const v = item[f.internalName];
if (v === undefined || v === null) continue;
if (isMultiChoice(f)) body[f.internalName] = { __metadata: { type: "Collection(Edm.String)" }, results: Array.isArray(v) ? v : (v.results || []) };
else if (isUrl(f)) body[f.internalName] = { __metadata: { type: "SP.FieldUrlValue" }, Url: v.Url, Description: v.Description || v.Url };
else body[f.internalName] = v;
}
try {
const created = await send(`${web}/_api/web/lists/getbytitle('${esc(list.title)}')/items`, body);
idMap[list.title][item.__id] = created.Id;
// Remember the new signature so duplicate rows in the source file are
// also de-duplicated within this same run.
if (SKIP_EXISTING_ITEMS) existing.set(signatureOf(scalarFields, item), created.Id);
added++;
} catch (e) {
console.warn(`[Import] ! item (src #${item.__id}) in ${list.title} failed: ${e.message}`);
}
}
if (SKIP_EXISTING_ITEMS) console.log(`[Import] ${list.title}: +${added} added, ${skipped} skipped (already existed)`);
}
// ============ PASS 4: re-link lookups + people ============
for (const list of data.lists) {
const lookupFields = list.fields.filter((f) => (f.isLookup || f.isUser) && !f.unsupported);
if (!lookupFields.length) continue;
const info = await getListInfo(list.title);
console.log(`[Import] relink -> ${list.title}`);
for (const item of list.items) {
const newId = idMap[list.title][item.__id];
if (!newId) continue;
const body = { __metadata: { type: info.entityType } };
let any = false;
for (const f of lookupFields) {
const v = item[f.internalName];
if (v === undefined || v === null || (Array.isArray(v) && !v.length)) continue;
const n = f.internalName;
if (f.isLookup) {
const map = idMap[f.lookupListTitle] || {};
if (f.multi) {
const ids = v.map((x) => map[x.id]).filter(Boolean);
if (ids.length) { body[`${n}Id`] = { results: ids }; any = true; }
} else {
const id = map[v.id];
if (id) { body[`${n}Id`] = id; any = true; }
}
} else { // person
if (f.multi) {
const ids = [];
for (const x of v) { const id = await ensureUser(x.email); if (id) ids.push(id); }
if (ids.length) { body[`${n}Id`] = { results: ids }; any = true; }
} else {
const id = await ensureUser(v.email);
if (id) { body[`${n}Id`] = id; any = true; }
}
}
}
if (!any) continue;
try {
await send(
`${web}/_api/web/lists/getbytitle('${esc(list.title)}')/items(${newId})`,
body, "POST", { "X-HTTP-Method": "MERGE", "IF-MATCH": "*" }
);
} catch (e) {
console.warn(`[Import] ! relink item #${newId} in ${list.title} failed: ${e.message}`);
}
}
}
if (skippedLookups.length) {
console.warn(`[Import] ${skippedLookups.length} lookup column(s) were NOT created (target list missing in this run). Re-run the import once all lists exist to create them:`, skippedLookups);
}
console.log("[Import] DONE.");
})().catch((e) => console.error("[Import] FAILED:", e));
Yili Zhang
Posted on Edited on
Top comments (0)