Backport of fix: avoid sending HEAD request on provider download into v1.12 (#37073)

* backport of commit b7cdffa5fb

* backport of commit 1c26d96d07

---------

Co-authored-by: Or Shachar <orchoock@gmail.com>
Co-authored-by: Liam Cervante <liam.cervante@hashicorp.com>

.changes/v1.12/BUG FIXES-20250514-184357.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: Fix regression during provider installation by reverting back to not sending HEAD requests.
+time: 2025-05-14T18:43:57.332193-04:00
+custom:
+    Issue: "36998"

internal/providercache/package_install.go

@@ -34,6 +34,7 @@ func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targ
 		Client:                httpclient.New(),
 		Netrc:                 true,
 		XTerraformGetDisabled: true,
+		DoNotCheckHeadFirst:   true,
 	}
 
 	urlObj, err := url.Parse(urlStr)