From 5df6af1b2b53a623aac99355888d70702bb75b61 Mon Sep 17 00:00:00 2001 From: Dave Arnold Date: Wed, 12 Feb 2025 22:29:10 -0800 Subject: [PATCH] working on scripts --- aws-image-pipeline.code-workspace | 29 --- external-dependencies.md | 137 ++++++++++++ external-dependencies.tf | 25 +++ linux-images.code-workspace | 13 ++ morpheus.tf | 1 - moved.tf | 41 ++++ scripts/__pycache__/syncrepo.cpython-39.pyc | Bin 0 -> 5186 bytes scripts/__pycache__/syncrepos.cpython-39.pyc | Bin 0 -> 5187 bytes .../test_sync_repos.cpython-39.pyc | Bin 0 -> 3041 bytes scripts/sync-repos.py | 136 ----------- scripts/syncrepos.py | 211 ++++++++++++++++++ scripts/test_sync_repos.py | 68 ++++++ 12 files changed, 495 insertions(+), 166 deletions(-) delete mode 100644 aws-image-pipeline.code-workspace create mode 100644 external-dependencies.md create mode 100644 external-dependencies.tf create mode 100644 linux-images.code-workspace create mode 100644 moved.tf create mode 100644 scripts/__pycache__/syncrepo.cpython-39.pyc create mode 100644 scripts/__pycache__/syncrepos.cpython-39.pyc create mode 100644 scripts/__pycache__/test_sync_repos.cpython-39.pyc delete mode 100755 scripts/sync-repos.py create mode 100755 scripts/syncrepos.py create mode 100644 scripts/test_sync_repos.py diff --git a/aws-image-pipeline.code-workspace b/aws-image-pipeline.code-workspace deleted file mode 100644 index ba0cc0e..0000000 --- a/aws-image-pipeline.code-workspace +++ /dev/null @@ -1,29 +0,0 @@ -{ - "folders": [ - { - "path": ".", - "name": "aws-image-pipeline" - }, - { - "path": "../image-pipeline-ansible-playbooks", - "name": "ansible/ansible-playbooks" - }, - { - "path": "../linux-image-pipeline", - "name": "packer/linux-image-pipeline" - }, - { - "path": "../terraform-aws-image-pipeline", - "name": "modules/terraform-aws-image-pipeline" - }, - { - "path": "../image-pipeline-goss-testing", - "name": "test/goss-testing" - }, - { - "path": "../packer-plugin-amazon/docs" - } - ], - "settings": { - } -} diff --git a/external-dependencies.md b/external-dependencies.md new file mode 100644 index 0000000..fde19a6 --- /dev/null +++ b/external-dependencies.md @@ -0,0 +1,137 @@ +# External Dependencies Documentation + +This document lists all external dependencies that are not managed by Terraform in the aws-image-pipeline project. + +## Hardcoded Values + +### Build Configuration +- Builder Image: `aws/codebuild/standard:7.0` +- Terraform Version: `1.8.5` +- Packer Version: `1.10.3` (default) +- SSH User: `ec2-user` + +### AMI Configuration +- Base AMI ID: `ami-03fadeeea589a106b` +- Instance Type: `t2.micro` + +### Repository Names (Defaults) +- `linux-image-pipeline` +- `image-pipeline-ansible-playbooks` +- `image-pipeline-goss-testing` + +### Network Configuration +- Proxy Server: `proxy.tco.census.gov:3128` +- Allowed Domains: + - `.census.gov` + - `.eks.amazonaws.com` + - `.s3.amazonaws.com` + - `.amazonaws.com` + - `.gcr.io` + - `.pkg.dev` + - `downloads.morpheusdata.com` + - `169.254.169.254` + - Various internal network ranges (148.129.*, 10.*, 172.18-25.*) + +## External AWS Resources + +### AWS Parameter Store Values +- AMI configuration parameters +- Subnet configuration +- Security group settings +- Region settings +- Source AMI information +- Instance type configurations +- SSH user parameters +- Docker repository configurations +- AWS account ID parameters +- Shared accounts parameters +- Userdata parameters + +### S3 Bucket Artifacts +- Packer configurations (`linux-image-pipeline.zip`) +- Ansible playbooks (`image-pipeline-ansible-playbooks.zip`) +- Goss testing files (`image-pipeline-goss-testing.zip`) + +### AWS Secrets Manager +- WinRM credentials +- AWS credentials for build process +- SSH private keys + +### Security Groups +- Existing security group: `it-linux-base` + +### IAM Dependencies +- AWS managed policies (referenced in IAM policy documents) +- Cross-account roles and permissions + +### AWS Service Dependencies +- AWS Partition data (`aws_partition.current`) +- AWS Caller Identity (`aws_caller_identity.current`) +- AWS Region data (`aws_region.current`) +- KMS keys (referenced via ARNs) + +### VPC Dependencies +- Pre-existing Security Group IDs +- Pre-existing Subnet IDs +- Pre-existing VPC ID + +### State Backend Requirements +- Pre-existing S3 bucket for state storage +- Pre-existing DynamoDB table for state locking + +### Cross-Account Resources +- AMI sharing account IDs +- Cross-region S3 bucket replication configurations + +## Build Dependencies + +### Environment Variables +- HTTP_PROXY +- HTTPS_PROXY +- NO_PROXY + +### Source Control +- CodeCommit repository ARNs (when not using S3) +- Default branch names (defaulting to "main") + +### Build Resources +- Pre-existing ECR/Docker images + - `aws/codebuild/standard:7.0` + +### Configuration Files +- Ansible playbook: `hello-world.yaml` +- Goss profile: `base-test` + +## Workspace-Managed Resources +Resources that are created in this workspace but outside the terraform-aws-image-pipeline module: + +### S3 Resources +- Assets bucket (`aws_s3_bucket.assets_bucket`) - Used to store pipeline artifacts +- Associated bucket policies and access controls + +### IAM Resources +- Morpheus build user policy (`aws_iam_user_policy.morpheus_build_user`) +- AMI sharing policies and roles + +### Parameter Store Resources +- RHEL9 AMI parameters +- Base image parameters +- Ansible-related parameters + +### VPC Resources +- VPC endpoints for AWS services +- Associated security groups and routing configurations + +### Pipeline Configurations +- Multiple pipeline definitions: + - Amazon Linux pipeline + - RHEL pipeline + - Morpheus application pipeline + - Docker image pipeline + - GitHub runner pipeline + +### Volume Configurations +- Custom EBS volume mappings (e.g., for Morpheus deployments) + - Root volumes + - Application volumes + - Data volumes \ No newline at end of file diff --git a/external-dependencies.tf b/external-dependencies.tf new file mode 100644 index 0000000..934b2eb --- /dev/null +++ b/external-dependencies.tf @@ -0,0 +1,25 @@ +module "external_dependencies" { + source = "../terraform-aws-image-pipeline-external" + + project_name = "aws-image-pipeline" + assets_bucket_name = aws_s3_bucket.assets_bucket.bucket + state_bucket_name = local.state_config.bucket + + pipeline_iam_arns = [ + module.amazon_linux.iam_arn, + module.morpheus.iam_arn + ] + + vpc_config = { + vpc_id = local._vpc_config.vpc_id + region = local._vpc_config.region + security_group_ids = local._vpc_config.security_group_ids + subnets = local._vpc_config.subnets + } + + # Add common tags + tags = { + Project = "aws-image-pipeline" + Environment = local.environment + } +} \ No newline at end of file diff --git a/linux-images.code-workspace b/linux-images.code-workspace new file mode 100644 index 0000000..7424916 --- /dev/null +++ b/linux-images.code-workspace @@ -0,0 +1,13 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../terraform-aws-image-pipeline" + }, + { + "path": "../../terraform-aws-image-pipeline-external" + } + ] +} \ No newline at end of file diff --git a/morpheus.tf b/morpheus.tf index 0b6f06a..8f5b510 100644 --- a/morpheus.tf +++ b/morpheus.tf @@ -43,7 +43,6 @@ module "morpheus" { morpheus_version = "7.0.10-1", shutdown_behavior = "stop" } - assets_bucket_name = aws_s3_bucket.assets_bucket.bucket image_volume_mapping = [ { device_name = "/dev/sda1" # Root device diff --git a/moved.tf b/moved.tf new file mode 100644 index 0000000..9f20bbe --- /dev/null +++ b/moved.tf @@ -0,0 +1,41 @@ +# S3 Bucket moves +moved { + from = aws_s3_bucket.state_bucket + to = module.external_dependencies.aws_s3_bucket.state_bucket +} + +moved { + from = aws_s3_bucket.assets_bucket + to = module.external_dependencies.aws_s3_bucket.assets_bucket +} + +moved { + from = aws_s3_bucket_server_side_encryption_configuration.state_bucket_encryption["state_bucket"] + to = module.external_dependencies.aws_s3_bucket_server_side_encryption_configuration.state_bucket_encryption +} + +moved { + from = aws_s3_bucket_server_side_encryption_configuration.state_bucket_encryption["assets_bucket"] + to = module.external_dependencies.aws_s3_bucket_server_side_encryption_configuration.assets_bucket_encryption +} + +moved { + from = aws_s3_bucket_policy.assets_bucket_policy + to = module.external_dependencies.aws_s3_bucket_policy.assets_bucket_policy +} + +# Security group moves +moved { + from = aws_security_group.allow_amznlinux_cdn + to = module.external_dependencies.aws_security_group.pipeline_security_group +} + +moved { + from = aws_vpc_security_group_egress_rule.allow_all_traffic_ipv4 + to = module.external_dependencies.aws_vpc_security_group_egress_rule.allow_all_traffic_ipv4 +} + +moved { + from = aws_vpc_security_group_ingress_rule.allow_all_between_self + to = module.external_dependencies.aws_vpc_security_group_ingress_rule.allow_self_traffic +} \ No newline at end of file diff --git a/scripts/__pycache__/syncrepo.cpython-39.pyc b/scripts/__pycache__/syncrepo.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c59c66b37941d289c9e49b1bab9eae5de920d372 GIT binary patch literal 5186 zcmai2%X8bt8OP#95TqzsQ5?%oQY3YpFfl1NnZ8ty8^@2Noz~S@PMu(!;lNywf=mL` zE@(#>Fzt+dXb+jD=QuM`Zs|<_g`Rut8P}e2>r8Ly^zr)^q+XR;O1kEn>)H+4P9W-L!CiF-oH(4hDt!vq+|08a@0}(0CiK`~!+mBN}}@G&p<0xXz8oM4id4 z$C_`2Mren|vc~NvIH2ou{fZUN{Xm8t z9*M3FE@lta>Ai+mlGbvTYaJ*9%h@eqqo=7CrVIx?L1Oqy6IFb^xxMnf424|bfryg< zU+G8bO7KXwqG8YvTccmK=jAMFivGVdxK=@fmcs47Nv=ZLg_t<(hYAo z9;DG|5MpSUq@irGqQ=9LPZ(F`v>*){$@G?h=^4RbP}ta@AH}}0F?uBCK!5h~Z8gbD z6OUw5?)@4Mdja-e^{}Xk#g*2)7H3*Myk}7*DR)Lex>497OvfVblIxf~ zWSk~mrTjHipjZI>fgu*D)(6ycZC63Gujl%ne$SW~6LVtaFbd3iC)K%?Tc2xR=sS#Q zeV9OQk}2$(r8Eu=F0{O&2X7xoQ5JA%rvrRX5v3Im)xV#9dZ_w-Wqn`Un2R?oN#ahqCdZF8ngY^G&r39G4{ z+dGU#ZS6zgc4B{GuTQ_1v`eT#Ep~SG_Bi$ePr~R2`&uBxmS*Cm8=)6|49h3kT=Znh zA)bpK%|IPeYTtpS+hxD{lH^_m8-Wz=Pp^7Ml0o^q#N@d~#wk(gRO!pJ7QCE|WV&QSnynt>Z zh$1LJqcanbF;%Jm!>-h!xduDX+#$m`;@~YT-UkVJNI(Ln&e@mHm4bv$pn$e?#Lf%Q z6(e5=Xc)OUg9bOD1H4TudxHQ$;_Z%wfLHYHiy-c95IlmTo^iX~G{sq*m;uKRTMF4( zz10ft->0?+d%pSJjjPv7aJ+aF97lIw^eVdB&R8WXI z@IS>One}wXPD$_gFhj*VO(zsMI%IkFFba-NM+|H{O2h*>3c6vZ7Y)J^5h_SwD|$%E zM3F+oPft=n8TFcf4A5X(rCnr|(m% z#Miz&NV>s5UO{EH)UQnGRhe0dv`v`+Z-XS@Feat%?_=-6B#!+$BFf#wy4pdML#v)UGhlDsr1fwIbNlY#@Y=s%cH>~SqNcKI z?`fYg@j-4-i5c^X{)K(|X}?n^?T7g54U^i!qs z+|#6jJ@4@|x$}@w4Kyu3O+ufst$X||&~Pq$MP+}ljmCq)%4jS%n(%lMkzD0`$a5(j z+-_&5lqXyDu16y5QkR#_dptp&7$+%3C7IsL>h4+@Qz#S2jCQ-7&8=Z$YW74j^s*Wy z7*pq}+^BET$)&e^;V$xS%w6?v20#KQ@YCpYZDmRVoF+M73UEO>mRYUU8YQ9|4uUAo z&XpdwyAgID(6N1l%W~eTmn~h}2;x3y=3xpha6%M=kBchUMk;{;&9nUtB!&fRXG`Ul zXemBGnOyaJy9A9`1S#3s>8T1wDi7+GC*jcA8l*}q!3}&58ricq9)@B&RlVa;D@|HJ zdYGL%k$<^CEAJp^>xmRUB+ji!;>x1}h9IAR5JZ~QL-FGs8Y0)_H;y`fm6rXV@KU8j z9Vyl*xq%(z2Pnw7EHG1F&|QkzhQm(d>oWacE>hkGTbSiu`hu#vDi52@C|ifuxY(z` z7WM4Z%w6rDdBm}k`>EkoG3aloP_*CECWu25M0ng)H5G}pO=FKy^r7rb{vgHum7Gax z?b_VhqZ-?+>^eE-AyV1P+8(ku>{ZQ8xF#gc3Vb$tTwEA!j8?KUXw`B%uORuZ?-}yf zyfV@6YFj?Cyt(Z^AE zY@#n-L|=WsPtnX>z;rHq_2;l|uQ!gnbg3tcP#3A=7oPT=Ba)*?PPvi_i3A!I5*H~X z?&3PU#{`_~EHGyCZAvU#`!Nl%(g|ZO=_ZKF&??p!tamIRi65c+w|E8RVVO5mB^c7b<-2?;nbo+VV$5y(NN(J z$?&!;OiIg3Y?_-%6pO1g{t6Y8GZmFtXnhl{8Ae}HBU6(6`F*jgR6+`s5+DPgUKY!H zkx-caE^T})9W+VcI~b6!qtJ8*03~=j%r%zqI;@UVdqG`}me}(;jnpwKs~^@FejcI! zMf%wy4u)Z;Q@EYZFyZ3?)$5(k);JiHBjPP0gMe8|h%yGnGg2FfNf>AsRyxMd2Zez< zU+JZkzasGz=KUlYDEWvxwDf=>RdcD2jF(z+0R?=F;nbGqFU&7hm+Tj>s8OPnSnqUv zW9{zs=7Q3cxJ9d5R6Ik)6%>V?ZjWFnf>5ET6N+RyL%dHDDq;LE63ERA49&e?Jm>6=rPzmRi|Iqx+m-%^#EQ+a$n3-A>xS#oFR+0)bAUw03J zxj9$E^WJa%*q3KC?XT20{1|AwjaLd3T;pt@`Q?}SOtp1iS6{<7)YtS)eD#4fw0(P6 z@hd~ecbN8$#tm*h*0{N&`_<6p)&pI{p|)Aut#f;i`Ey}4oZDf3BW&!>?`aHst!vH7 z*VtFP-82gGM$}CU=NF?iO5$Kpm_LhT+NIIMPX&#)@yb7-2sPr-*F%G|CyeXdcud@x z+4ffp*>u!dPik7l_~3|`~)$Ch75pBh&y=Xe7v z=k}@1gW3W=g}w$qGwYqwo9By9O#TdC0+j{+EI$oCPVp+vxm;AMMAGSu(C>%s_*b?C=ktYnS$gYS+sa%m@PEBf$whO1ziQ*{jbchkD z*kswl^a$AkdFu-`E&E5_8}`JO7@V zV?TEkFEDB&BuHOyYWj)Q>P zX&(_C<9NM95TPT7XfkmM`!~(P!GT94hN5wGdUPcfQPcy@J2P<3gtV&yOj%(|B9+{&%bwJ-D? z#>rO3j7nGoBgazQqmJ$F+G zNWLGYX%zQo)KX#&KtAeT6j=QJfZ*OaT^rMuRAA+NGpT61#wR)e5tl z@USqFaatImI4=C6hW$EIo22=&fkHD}<`}1;hz2u_?3tPP56vb2Xmp1h#Un13vF3>I z33a4gCHzfPvNs8PIYX(_o9haP*2KsS66!s@6#b@|o4M69?n2zAmRj4KX%m}i*;&GB zYUlP2qfuM?5V)P#pV;fu?AdzkGr7?DTM*uH7iPG;uVlAx2R0%iPxyBNn5e-IA~r# zH!(yZ6rjd;(+9cb>5;T-Ys78dV=gghi50aNGfOXx~LLMKo_TRMX0 zE6^1qUkGRzxjBOdH=zT(O)GnY072sIj)j0%^zMrw?rsn~f}@^syWKR!S?rhr#}8Wy z*;&2S3hv*hwg`K^`R>iDH%f54coZB*sjqslMnMu0-=gA06o3r*9EAaKo@(Ew;sO;E zqYnH}u}Ee;rPwL?{T^nhfT!t%0!N1|&mKm>(dh_+jYo-iAV)zr?DV2RSRz6NDUhOv zq)Z$sMEn$$6b^~A9sPF~Cwj_u$px{Ap;88oT+p(1L)&3@5rDa6V)+hY(Yp2>OJ&1$wygbL-y&tR!X)3T1PKA??MuF0QamnT_cHAWJdCc;>Hm&b%P!5tS^D&S zZk71jmj_8V7|1KA%$E9$Yj4TDB^GfK!YKuKCA2lJkf@sz)i|)G$Yq|@ zxWlWtm79H9Nm0(8R3=Vta~DzW7S`1cq8wWF+?fG;>lUqtJDA&d2Zz`G1+yCmvlTa$ zU3*XajEN6&gG$VpSNt#R^H20nokS1w&tv}LF^c60kiq80Nhg@@^6aryB$b|$%IT*{ z<+-Ox1vKyRGr9ASQ4KsTKTSfPv8{XjEYNT+ds$_FuZ_lo!OCbXH=6Kx5|Lcxd`NUD z9o%kbr<5mK^=?EW>{6GP&3imSo){-7MJ1Wu%IfY~8B-_|$c%Qooz1O*Fg1H38G2cb z5{#*HRc_QbY3I^gzHk?bH|DN-w*nx66ZmO#y0$W90Zx+~Fa@|E9m}lNYK;=n4F^FK zXXi?f+uaDe4`|yy!eu#c)ytNyZv=54Jo7NcDX>En;~W=Nu#Hp#1DEqgI-< zfb=jscjEcw2Cck=A z!4~!G)XZJ&?|HR6GV7iRy7rgv`u4=QS_ngO#UFn1(uvi zYVF$G+M^oVtn4~D<{?tq%i12YH_)o)CR`JeW(7VQJuYsHHbyJi8MJD-omY^2*Y^zh zYhIb?ceO1aS>D|CpOEF@%AQ@oUT{rtU{wWZwQo5S`JOv*K5@4E@=N*NmY?gQz?#}V ze>U+KFQTu$-=}!yE?_#Bz4~)lx7QoTUAoniMW~C^@e5D;&JoE`B&S?Sg+v043WJ-Me6Yto5WyJ3#V&_0yK-kLfjZ<=%xzA_ydRn~N` z9=in2<^b+;+@+Ma2dA?AQg*>zPlj~0ow_-?_4W=tPwC2<4RzBK?_$@YLSdbtN6}E> z56SShEKEwvOKh5(Nfe8#H2yLblrt5TS!jI|tr{w@(dmJXUE@Er`u*HLJ?1Ar1d9p)NKcpX+ps=c5tM@#H^okr@ImDLYx48M=i z|04Zt5eLJt(<$6eXPEHufa>*5XKNe`$`SDvu|dErB}5s6Ix|umh)Eb|7gjpP&j*Ep zJ74Lgl)obJ6z2UT87TRPJGAuRK&s|a9~m#TlQ#yL5 z;~Q&tZ!{N_ro?Sp-J;?dDz2a?>~wnsLlJ}u#hp+r(;nh|notSjhmk;TW+3NOIxZXp jkPboZsXP|W1Aa literal 0 HcmV?d00001 diff --git a/scripts/__pycache__/test_sync_repos.cpython-39.pyc b/scripts/__pycache__/test_sync_repos.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e5b083bd28a162cbd6e85d43913b0638eb1bc13 GIT binary patch literal 3041 zcma)8OOF#r5bmCr$Kx^VE=zcmjR^@BtPLoq2oWTNP!9MIKoJuqliBIE8Sgxt?r{L) zJuRn51S#hnyvIG{FXV^JH7EW8r&RTLy=H9^+M1f`?y8>dufDEwP_H`#TJN{NdRJXS z{=v?CGoW(^9{CdpMi`BWhi@&WiRNjE?&*o)8MK@;JyY3Oo&|3`t|Yc+Q*xg$gPE@g zGdby3cXiL?p5c%$JW>hHR`Z+K;vXl zd&Fgaa8wE>I%ueC@Z5z*egcBgG4V+bvJcCGt}y~jXr0lqb~o7O?X5uaHj4ycYoqRi zE==$&>|T`n62Jsp4awu~J`qTXcTd`nB^R>I0+D7hYlB@ocqTWaB2q;B)Ke{hc9H# zlY7sjT;OgBBMG9ksFgZvmqT9~M4 zl%j^l=MCmqFdCZSLo==d079$b5!3NMsXgcvW53; z^v%mg7orXXXnei#!^;N70klR*CyeBc#?G~kWfM0WJ3sC;HP21?Gd~RCIM|B0R~0;9 zJ`3_-bP?$`Wy6%-Eip{}FiVmkWuwbW=6gK%!=Vs7&HXJAq~X)h!gEd=UVMa7ao|r- z812!q;X|Stn;>;$1T)4O-jcqCd|+m7FpE`~{aVB8{^n5`|J+iXu@q-2OF8rN5zg3S zt6%Lq%t6j9FzokvYsIKq=iTX9;9AIp%~|)6ScUEH>jF7f5OEG)&L9wQX#>%dg${_m zt6?ZN5U?oU*E~z`B+GeGNw}17J@=_N27YRdX06mQVH8%j)#qW!P=Oam97nN&0>3NP zP@F)q4r0He2>aqC6Md{PU8P`S$t+ytJvvR#Cj4PgL@x=9_hsZ9rm1rD z3m9#h1Y-OF68JSw-N%3#-gKM`1dp*u1#hSKuQtArlRAv8&|h z(gL3sJs04@LapH100lS;l@01s2^4Ud0aQJ0R List[str]: - """Get list of remotes based on command line argument.""" - return ['origin', 'hpw'] if args.remote == 'all' else [args.remote] - -def run_command(cmd: list[str], cwd: Optional[str] = None) -> tuple[int, str, str]: - """Run a shell command and return the exit code, stdout, and stderr.""" - try: - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=cwd, - text=True - ) - stdout, stderr = process.communicate() - return process.returncode, stdout, stderr - except Exception as e: - return 1, "", str(e) - -def sync_repo(repo_name: str, config: dict, remotes: List[str], commit_message: Optional[str] = None) -> bool: - """Sync (pull/push) a single repository.""" - repo_path = os.path.join(BASE_DIR, repo_name) - if not os.path.isdir(repo_path): - print(f"Error: Directory {repo_path} does not exist") - return False - - print(f"\nProcessing {repo_name}...") - - # Pull from specified remotes - for remote in remotes: - print(f"Pulling from {remote}...") - code, out, err = run_command( - ["git", "pull", remote, config["branch"]], - repo_path - ) - if code != 0: - print(f"Warning: Failed to pull from {remote}") - print(f"Error: {err}") - - # Check for changes - code, out, err = run_command(["git", "status", "--porcelain"], repo_path) - if code != 0: - print(f"Error checking git status: {err}") - return False - - if out.strip() and commit_message: - print(f"Changes detected in {repo_name}, committing...") - - # Add all changes - code, out, err = run_command(["git", "add", "."], repo_path) - if code != 0: - print(f"Error adding files: {err}") - return False - - # Commit changes - code, out, err = run_command( - ["git", "commit", "-m", commit_message], - repo_path - ) - if code != 0: - print(f"Error committing changes: {err}") - return False - - # Always try to push to specified remotes - for remote in remotes: - print(f"Pushing to {remote}...") - code, out, err = run_command( - ["git", "push", remote, config["branch"]], - repo_path - ) - if code != 0: - # Only warn if the error indicates nothing to push - if "Everything up-to-date" not in err: - print(f"Warning: Failed to push to {remote}") - print(f"Error: {err}") - - return True - -def main(): - """Main function to sync all repositories.""" - args = parse_args() - remotes = get_remotes(args) - success = True - - # Process each repository - for repo_name, config in DEFAULT_REPOS.items(): - try: - if not sync_repo(repo_name, config, remotes, args.message): - success = False - except Exception as e: - print(f"Error processing {repo_name}: {e}") - success = False - - if success: - print("\nAll repositories processed successfully!") - sys.exit(0) - else: - print("\nSome repositories failed to process") - sys.exit(1) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/syncrepos.py b/scripts/syncrepos.py new file mode 100755 index 0000000..b9c33e5 --- /dev/null +++ b/scripts/syncrepos.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess +import argparse +import json +from typing import Dict, Optional, List + +class GitSync: + def __init__(self, base_dir: str): + self.base_dir = base_dir + + def parse_args(self): + """Parse command line arguments.""" + parser = argparse.ArgumentParser(description='Sync git repositories with multiple remotes') + parser.add_argument('--base-dir', + default=os.environ.get("PWD"), + help='Base directory for repositories') + parser.add_argument('--remote', '-r', + choices=['all', 'origin', 'hpw'], + default='all', + help='Remote to sync with (default: all)') + parser.add_argument('--message', '-m', + help='Commit message to use for all repositories') + parser.add_argument('--workspace-file', + help='Path to VS Code workspace file') + parser.add_argument('--repo-paths', + help='Comma-separated list of repository paths') + parser.add_argument('--branch', + help='Branch to checkout before syncing') + return parser.parse_args() + + def get_remotes(self, args, repo_path: str) -> List[str]: + """Get list of remotes based on command line argument.""" + # If a specific remote is specified, return it as a single-element list + if args.remote != 'all': + return [args.remote] + + # Otherwise, get all configured remotes for the repository + code, out, err = self.run_command(["git", "remote"], repo_path) + if code != 0: + print(f"Error getting remotes: {err}") + return [] + + # Return the list of remotes + return out.strip().split('\n') + + def run_command(self, cmd: list[str], cwd: Optional[str] = None) -> tuple[int, str, str]: + """Run a shell command and return the exit code, stdout, and stderr.""" + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + text=True + ) + stdout, stderr = process.communicate() + return process.returncode, stdout, stderr + except Exception as e: + return 1, "", str(e) + + def create_branch(self, branch_name: str, repo_path: str) -> bool: + """Create a new branch in a git repository.""" + code, out, err = self.run_command(["git", "checkout", "-b", branch_name], repo_path) + if code != 0: + print(f"Error creating branch: {err}") + return False + + def get_current_branch(self, repo_path: str) -> Optional[str]: + """Get the current branch of a git repository.""" + code, out, err = self.run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"], repo_path) + if code == 0: + return out.strip() + else: + print(f"Error getting current branch: {err}") + return None + + def parse_workspace_file(self, workspace_file: str) -> List[str]: + """Parse the VS Code workspace file to get the list of folders.""" + with open(workspace_file, 'r') as file: + workspace_data = json.load(file) + repo_paths = [] + for folder in workspace_data['folders']: + path = folder['path'] + if path == ".": + path = os.getcwd() + repo_paths.append(path) + return repo_paths + + def is_git_repo(self, repo_path: str) -> bool: + """Check if a directory is a Git repository.""" + return os.path.isdir(os.path.join(repo_path, ".git")) + + def sync_repo(self, repo_path: str, remotes: List[str], commit_message: Optional[str] = None) -> bool: + """Sync (pull/push) a single repository.""" + if not os.path.isdir(repo_path): + print(f"Error: Directory {repo_path} does not exist") + return False + + if not self.is_git_repo(repo_path): + print(f"Error: Directory {repo_path} is not a Git repository") + return False + + print(f"\nProcessing {repo_path}...") + + current_branch = self.get_current_branch(repo_path) + if not current_branch: + return False + + # Pull from specified remotes + for remote in remotes: + print(f"Pulling from {remote} on branch {current_branch}...") + code, out, err = self.run_command( + ["git", "pull", remote, current_branch], + repo_path + ) + if code != 0: + print(f"Warning: Failed to pull from {remote}") + print(f"Error: {err}") + + # Check for changes + code, out, err = self.run_command(["git", "status", "--porcelain"], repo_path) + if code != 0: + print(f"Error checking git status: {err}") + return False + + if out.strip() and commit_message: + print(f"Changes detected in {repo_path}, committing...") + + # Add all changes, including deletions + code, out, err = self.run_command(["git", "add", "-A"], repo_path) + if code != 0: + print(f"Error adding files: {err}") + return False + + # Commit changes + code, out, err = self.run_command( + ["git", "commit", "-m", commit_message], + repo_path + ) + if code != 0: + print(f"Error committing changes: {err}") + return False + + # Always try to push to specified remotes + for remote in remotes: + print(f"Pushing to {remote} on branch {current_branch}...") + code, out, err = self.run_command( + ["git", "push", remote, current_branch], + repo_path + ) + if code != 0: + # Only warn if the error indicates nothing to push + if "Everything up-to-date" not in err: + print(f"Warning: Failed to push to {remote}") + print(f"Error: {err}") + + return True + + def main(self): + """Main function to sync all repositories.""" + args = self.parse_args() + success = True + self.base_dir = args.base_dir + + # Determine repository paths + if args.repo_paths: + repo_paths = args.repo_paths.split(',') + else: + # Find the VS Code workspace file + workspace_file = args.workspace_file + if not workspace_file: + workspace_files = [f for f in os.listdir(self.base_dir) if f.endswith('.code-workspace')] + if not workspace_files: + print("Error: No .code-workspace file found and no repo paths provided") + sys.exit(1) + workspace_file = os.path.join(self.base_dir, workspace_files[0]) + repo_paths = self.parse_workspace_file(workspace_file) + + # Process each repository + for repo_path in repo_paths: + remotes = self.get_remotes(args, repo_path) + try: + if args.branch: + current = self.get_current_branch(repo_path) + if current != args.branch: + print(f"Checking out branch {args.branch}...") + code, out, err = self.run_command(["git", "checkout", args.branch], repo_path) + if code != 0: + print(f"Error checking out branch: {err}") + success = False + continue + + if not self.sync_repo(repo_path, remotes, args.message): + success = False + except Exception as e: + print(f"Error processing {repo_path}: {e}") + success = False + + if success: + print("\nAll repositories processed successfully!") + sys.exit(0) + else: + print("\nSome repositories failed to process") + sys.exit(1) + +if __name__ == "__main__": + base_dir = os.environ.get("PWD") + git_sync = GitSync(base_dir) + git_sync.main() \ No newline at end of file diff --git a/scripts/test_sync_repos.py b/scripts/test_sync_repos.py new file mode 100644 index 0000000..6cba7bd --- /dev/null +++ b/scripts/test_sync_repos.py @@ -0,0 +1,68 @@ +import unittest +from unittest.mock import patch, mock_open, MagicMock +import os +import json +from syncrepos import GitSync + +class TestGitSync(unittest.TestCase): + def setUp(self): + self.base_dir = "/fake/base/dir" + self.git_sync = GitSync(self.base_dir) + + @patch("os.path.isdir") + @patch("os.listdir") + def test_main_no_workspace_file(self, mock_listdir, mock_isdir): + mock_listdir.return_value = [] + with self.assertRaises(SystemExit) as cm: + self.git_sync.main() + self.assertEqual(cm.exception.code, 1) + + @patch("os.path.isdir") + @patch("os.listdir") + @patch("builtins.open", new_callable=mock_open, read_data='{"folders": [{"path": "/fake/repo1"}, {"path": "/fake/repo2"}]}') + @patch("syncrepos.GitSync.run_command") + @patch("syncrepos.GitSync.get_current_branch") + def test_main_success(self, mock_get_current_branch, mock_run_command, mock_open, mock_listdir, mock_isdir): + mock_listdir.return_value = ["workspace.code-workspace"] + mock_isdir.return_value = True + mock_get_current_branch.return_value = "main" + mock_run_command.return_value = (0, "", "") + + with patch.object(self.git_sync, 'parse_args', return_value=MagicMock(remote='all', message='test commit')): + with self.assertRaises(SystemExit) as cm: + self.git_sync.main() + self.assertEqual(cm.exception.code, 0) + + @patch("os.path.isdir") + @patch("os.listdir") + @patch("builtins.open", new_callable=mock_open, read_data='{"folders": [{"path": "/fake/repo1"}, {"path": "/fake/repo2"}]}') + @patch("syncrepos.GitSync.run_command") + @patch("syncrepos.GitSync.get_current_branch") + def test_main_failure(self, mock_get_current_branch, mock_run_command, mock_open, mock_listdir, mock_isdir): + mock_listdir.return_value = ["workspace.code-workspace"] + mock_isdir.return_value = True + mock_get_current_branch.return_value = "main" + mock_run_command.side_effect = [(0, "", ""), (0, "", ""), (1, "", "error")] + + with patch.object(self.git_sync, 'parse_args', return_value=MagicMock(remote='all', message='test commit')): + with self.assertRaises(SystemExit) as cm: + self.git_sync.main() + self.assertEqual(cm.exception.code, 1) + + @patch("syncrepos.GitSync.run_command") + def test_get_current_branch(self, mock_run_command): + mock_run_command.return_value = (0, "main", "") + branch = self.git_sync.get_current_branch("/fake/repo") + self.assertEqual(branch, "main") + + mock_run_command.return_value = (1, "", "error") + branch = self.git_sync.get_current_branch("/fake/repo") + self.assertIsNone(branch) + + @patch("builtins.open", new_callable=mock_open, read_data='{"folders": [{"path": "/fake/repo1"}, {"path": "/fake/repo2"}]}') + def test_parse_workspace_file(self, mock_open): + repo_paths = self.git_sync.parse_workspace_file("/fake/workspace.code-workspace") + self.assertEqual(repo_paths, ["/fake/repo1", "/fake/repo2"]) + +if __name__ == "__main__": + unittest.main()